From ed5a162de2e07c8d33f4dcac2bebd5a230e8aa84 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 16:58:42 +0200 Subject: [PATCH 01/21] Add endpoint agent v0 checkpoint --- docker/server/release/Dockerfile | 2 + docker/server/stgn/Dockerfile | 2 + endpoint-agent-backend-troubleshooting.md | 534 +++++++ endpoint-agent-checkpoints.md | 115 ++ endpoint-agent-harness-test-plan.md | 729 +++++++++ qwen-endpoint-happy.dstack.yml | 10 + src/dstack/_internal/cli/commands/endpoint.py | 149 ++ src/dstack/_internal/cli/commands/logs.py | 88 +- src/dstack/_internal/cli/main.py | 2 + .../_internal/cli/services/completion.py | 12 + .../cli/services/configurators/__init__.py | 2 + .../cli/services/configurators/endpoint.py | 431 +++++ src/dstack/_internal/cli/services/profile.py | 3 +- src/dstack/_internal/cli/utils/endpoint.py | 91 ++ .../_internal/core/models/configurations.py | 5 + src/dstack/_internal/core/models/endpoints.py | 154 ++ src/dstack/_internal/core/models/events.py | 1 + src/dstack/_internal/server/app.py | 3 + .../background/pipeline_tasks/__init__.py | 2 + .../background/pipeline_tasks/endpoints.py | 901 +++++++++++ .../07_03_1243_03efb71a1563_add_endpoints.py | 127 ++ src/dstack/_internal/server/models.py | 68 + .../_internal/server/routers/endpoints.py | 132 ++ .../_internal/server/schemas/endpoints.py | 34 + .../server/services/endpoints/__init__.py | 573 +++++++ .../services/endpoints/agent/__init__.py | 91 ++ .../server/services/endpoints/agent/claude.py | 977 ++++++++++++ .../server/services/endpoints/agent/report.py | 48 + .../agent/resources/deployment_harness.md | 42 + .../dstack_cli_and_service_authoring.md | 25 + .../agent/resources/recipes_guide.md | 24 + .../agent/resources/system_prompt.md | 13 + .../server/services/endpoints/names.py | 12 + .../server/services/endpoints/planning.py | 170 ++ .../services/endpoints/preset_building.py | 110 ++ .../server/services/endpoints/presets.py | 339 ++++ .../_internal/server/services/events.py | 9 + src/dstack/_internal/server/settings.py | 26 + src/dstack/_internal/server/testing/common.py | 8 +- src/dstack/api/server/__init__.py | 6 + src/dstack/api/server/_endpoints.py | 49 + src/tests/_internal/cli/commands/test_logs.py | 124 ++ .../services/configurators/test_endpoint.py | 214 +++ .../_internal/cli/utils/test_endpoint.py | 171 ++ .../_internal/core/models/test_endpoints.py | 62 + .../pipeline_tasks/test_endpoints.py | 1386 +++++++++++++++++ .../server/routers/test_endpoints.py | 521 +++++++ .../services/endpoints/test_agent_report.py | 60 + .../services/endpoints/test_claude_agent.py | 683 ++++++++ .../server/services/test_endpoint_presets.py | 784 ++++++++++ 50 files changed, 10109 insertions(+), 15 deletions(-) create mode 100644 endpoint-agent-backend-troubleshooting.md create mode 100644 endpoint-agent-checkpoints.md create mode 100644 endpoint-agent-harness-test-plan.md create mode 100644 qwen-endpoint-happy.dstack.yml create mode 100644 src/dstack/_internal/cli/commands/endpoint.py create mode 100644 src/dstack/_internal/cli/services/configurators/endpoint.py create mode 100644 src/dstack/_internal/cli/utils/endpoint.py create mode 100644 src/dstack/_internal/core/models/endpoints.py create mode 100644 src/dstack/_internal/server/background/pipeline_tasks/endpoints.py create mode 100644 src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py create mode 100644 src/dstack/_internal/server/routers/endpoints.py create mode 100644 src/dstack/_internal/server/schemas/endpoints.py create mode 100644 src/dstack/_internal/server/services/endpoints/__init__.py create mode 100644 src/dstack/_internal/server/services/endpoints/agent/__init__.py create mode 100644 src/dstack/_internal/server/services/endpoints/agent/claude.py create mode 100644 src/dstack/_internal/server/services/endpoints/agent/report.py create mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md create mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md create mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md create mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md create mode 100644 src/dstack/_internal/server/services/endpoints/names.py create mode 100644 src/dstack/_internal/server/services/endpoints/planning.py create mode 100644 src/dstack/_internal/server/services/endpoints/preset_building.py create mode 100644 src/dstack/_internal/server/services/endpoints/presets.py create mode 100644 src/dstack/api/server/_endpoints.py create mode 100644 src/tests/_internal/cli/commands/test_logs.py create mode 100644 src/tests/_internal/cli/services/configurators/test_endpoint.py create mode 100644 src/tests/_internal/cli/utils/test_endpoint.py create mode 100644 src/tests/_internal/core/models/test_endpoints.py create mode 100644 src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py create mode 100644 src/tests/_internal/server/routers/test_endpoints.py create mode 100644 src/tests/_internal/server/services/endpoints/test_agent_report.py create mode 100644 src/tests/_internal/server/services/endpoints/test_claude_agent.py create mode 100644 src/tests/_internal/server/services/test_endpoint_presets.py diff --git a/docker/server/release/Dockerfile b/docker/server/release/Dockerfile index a88439d61e..df5de20fbe 100644 --- a/docker/server/release/Dockerfile +++ b/docker/server/release/Dockerfile @@ -22,6 +22,8 @@ ADD https://astral.sh/uv/install.sh /uv-installer.sh RUN sh /uv-installer.sh && rm /uv-installer.sh ENV PATH="/root/.local/bin/:$PATH" +RUN uv run --no-project --with claude-agent-sdk==0.2.110 python -c "from pathlib import Path; import claude_agent_sdk, shutil; shutil.copy2(Path(claude_agent_sdk.__file__).parent / '_bundled' / 'claude', '/usr/local/bin/claude')" && \ + chmod 755 /usr/local/bin/claude RUN uv tool install "dstack[all]==$VERSION" COPY entrypoint.sh entrypoint.sh diff --git a/docker/server/stgn/Dockerfile b/docker/server/stgn/Dockerfile index b2c4f96370..c8757808da 100644 --- a/docker/server/stgn/Dockerfile +++ b/docker/server/stgn/Dockerfile @@ -20,6 +20,8 @@ ADD https://astral.sh/uv/install.sh /uv-installer.sh RUN sh /uv-installer.sh && rm /uv-installer.sh ENV PATH="/root/.local/bin/:$PATH" +RUN uv run --no-project --with claude-agent-sdk==0.2.110 python -c "from pathlib import Path; import claude_agent_sdk, shutil; shutil.copy2(Path(claude_agent_sdk.__file__).parent / '_bundled' / 'claude', '/usr/local/bin/claude')" && \ + chmod 755 /usr/local/bin/claude COPY pyproject.toml uv.lock README.md ./ COPY src src RUN uv sync --extra all diff --git a/endpoint-agent-backend-troubleshooting.md b/endpoint-agent-backend-troubleshooting.md new file mode 100644 index 0000000000..92815bf9cd --- /dev/null +++ b/endpoint-agent-backend-troubleshooting.md @@ -0,0 +1,534 @@ +# Endpoint Agent Backend Troubleshooting + +This file is for concrete issues found while testing endpoint agent deployments. Keep it +evidence-first: record exact commands, dstack events, native backend observations, and +whether the issue was reproduced independently. + +## Diagnostic Workflow + +Before a live development agent run on a new or suspect hardware path, run an +independent dstack/backend preflight so agent quality is not judged against unknown +capacity: + +1. Check offers with the intended endpoint constraints. +2. For VM-based backends, optionally create/update the fleet separately and wait until + capacity can be pre-provisioned or reused. +3. For container-based backends such as RunPod and Vast.ai, do not treat fleet creation + as pre-provisioning. Use a pinned `nodes: 0..1` fleet template or direct run + constraints, then submit a tiny detached task to force real provisioning. +4. Submit a minimal detached task on the same backend/region/instance class, such as + `nvidia-smi`, with a short `max_duration`. +5. If this preflight fails before image pull/logs, diagnose and record it here as a + dstack/backend issue before running or blaming the endpoint agent. + +When an endpoint agent candidate is stuck or fails, collect evidence in this order. +Prefer the normal dstack control-plane path first; use native provider APIs only when +dstack says an instance exists but the instance cannot be reached or inspected through +the project SSH path. + +1. Endpoint state + + ```bash + uv run dstack endpoint get --json + uv run dstack logs --since 10m + ``` + +2. Backing run state + + ```bash + uv run dstack run get --json + uv run dstack event list --within-run --since 30m + uv run dstack logs --since 30m + ``` + +3. Fleet and instance state + + ```bash + uv run dstack fleet list + uv run dstack event list --within-fleet --since 30m + ``` + +4. Network and SSH reachability + + Check TCP/SSH from the same host where the dstack server runs. Use the project SSH + key that dstack used for the run. Never copy project private keys into the report. + + ```bash + python3 - <<'PY' + import socket, time + host = "" + port = 22 + start = time.time() + try: + s = socket.create_connection((host, port), timeout=8) + print("tcp_connect_ok", host, port, "elapsed", round(time.time() - start, 2)) + s.close() + except Exception as e: + print("tcp_connect_failed", type(e).__name__, str(e), "elapsed", round(time.time() - start, 2)) + PY + ``` + +5. If SSH works, inspect host bootstrap logs + + Start with: + + ```bash + ssh @ 'sudo systemctl status dstack-shim --no-pager || true' + ssh @ 'sudo journalctl -u dstack-shim -n 200 --no-pager || true' + ssh @ 'sudo tail -n 200 /var/log/cloud-init-output.log || true' + ssh @ 'docker ps -a || true' + ssh @ 'nvidia-smi || true' + ``` + +6. Native backend state, only when needed + + Use the existing backend implementation or provider API client only when dstack has + allocated an instance but SSH/shim is unreachable, or when dstack has no hostname and + the provider state is required to understand why. Do not print API keys or credentials. + For CloudRift, use `RiftClient.get_instance_by_id()` and print only sanitized fields + such as status, node mode, host address, VM readiness, and VM state. + +7. Candidate policy + + If a paid candidate remains in backend provisioning with no image pull and no logs for + several polls, stop it and try a different backend or offer. Do not let the agent sit + in a long shell sleep loop. + +## Issue Reports + +### EP-BACKEND-2026-07-04-001: CloudRift VM active in provider API, but dstack run stuck before image pull + +Status: Reproduced with separate minimal tasks. + +Endpoint: + +- Endpoint name: `qwen-endpoint-happy` +- Endpoint id: `3383f4f4-0992-4da2-b9a0-7450a2e9d7b6` +- Agent workspace: `~/.dstack/server/data/endpoint_agent_runs/3383f4f4-0992-4da2-b9a0-7450a2e9d7b6/workspace` + +Candidate run: + +- Run name: `qwen-endpoint-happy-serving` +- Service image: `vllm/vllm-openai:latest` +- Model: `Qwen/Qwen3-0.6B` +- Requested resources: `gpu: 24GB..:1`, `disk: 30GB..` +- Constraints: `spot_policy: on-demand`, `max_price: 0.5` + +Timeline: + +- `2026-07-04 15:29:11` run submitted. +- `2026-07-04 15:29:11` job created. +- `2026-07-04 15:29:11` instance `quick-duck-0` created, status `PENDING`. +- `2026-07-04 15:29:15` job moved `SUBMITTED -> PROVISIONING`. +- `2026-07-04 15:29:15` instance moved to `PROVISIONING`. +- `2026-07-04 15:29:18` run moved `SUBMITTED -> PROVISIONING`. +- No image pull progress appeared. +- No backing service logs appeared. +- `2026-07-04 15:38:14` run was manually stopped to avoid further spend. +- Final run cost reported by dstack: `$0.0592`. + +Actual dstack provisioning data: + +- Backend: `cloudrift` +- Region: `ap-east-tw-kn-1` +- Instance type: `rtx49-10c-kn.1` +- GPU: `RTX4090:24GB:1` +- CPU: `7` +- Memory: `48001 MiB` +- Disk: `1024 GiB` +- Price: `$0.39/hr` +- Host address reported to dstack: `211.21.50.85` +- SSH username: `riftuser` + +Native CloudRift API observation: + +- Instance id: `59aa1b06-77ac-11f1-9b61-937d98c38d58` +- CloudRift status: `Active` +- Node mode: `VirtualMachine` +- Host address: `211.21.50.85` +- VM `ready`: `true` +- VM state: `Running` + +Network observation from the server host: + +```text +tcp_connect_failed timeout timed out elapsed 8.02 +``` + +SSH observation: + +```text +ssh: connect to host 211.21.50.85 port 22: Operation timed out +``` + +What this rules out: + +- This was not a vLLM startup error: the container never reached image pull/startup. +- This was not a model download or dependency error: there were no job logs. +- This was not an endpoint verification failure: the service never became reachable. + +Working hypothesis: + +CloudRift marked the VM active/ready and returned a host address, but the dstack server +host could not reach TCP/22 on that address. The failure is likely in the provider +network path, VM SSH exposure, or early bootstrap path before dstack shim/runner became +reachable. + +Latest independent repro: + +- Repro run: `cloudrift-rtx49-provisioning-smoke` +- Repro time: `2026-07-04 15:58:29` to `2026-07-04 16:00:53` +- Final dstack status: `terminated`, `stopped_by_user` +- Final cost reported by dstack: `$0.015` +- Instance id: `70bb964a-77b0-11f1-bc41-ab4db0e59660` +- Backend/region/instance: `cloudrift`, `ap-east-tw-kn-1`, `rtx49-10c-kn.1` +- Native state while dstack was stuck in provisioning: CloudRift `Active`, + `node_status=Ready`, `node_mode=VirtualMachine`, VM `ready=true`, + VM `state=Running` +- Native host fields: `host_address=211.21.50.85`, + `internal_host_address=10.21.106.99` +- dstack provisioning data after `vm_ready=true`: `hostname=211.21.50.85`, + `ssh_port=22`, `username=riftuser` +- TCP/22 from the dstack server host timed out repeatedly after `vm_ready=true` +- SSH could not be attempted beyond TCP connect because port `22` was unreachable +- `dstack logs cloudrift-rtx49-provisioning-smoke --since 30m` returned no workload + logs +- `image_pull_progress` stayed `null` + +Root-cause status: + +- Reproduced: yes. +- dstack-side behavior diagnosed: yes. +- Provider/backend-side failing boundary: confirmed. CloudRift reported a running, + ready VM and returned a public host address, but the dstack server host could not + reach `host_address:22`. +- Exact provider-side cause is still one level deeper: NAT/firewall/host-address mapping + vs guest SSH/cloud-init. Because SSH never became reachable, host-local logs could not + be inspected from dstack. + +dstack-side diagnosis: + +- `CloudRiftCompute.update_provisioning_data()` treats the VM as provisioned enough to + try SSH once CloudRift returns `virtual_machines[0].ready == true`. +- It then sets `job_provisioning_data.hostname = instance_info["host_address"]` and + leaves `ssh_port = 22`, `username = riftuser`. +- After that, the instance check path attempts an SSH tunnel to the shim through + `riftuser@:22`. +- In this failure, TCP/22 to `211.21.50.85` timed out from the dstack server host, so + dstack could not reach the shim and the run stayed in provisioning until it was + manually stopped. +- This means the endpoint/service/agent code was not the failing layer. The failure + happened between CloudRift reporting VM readiness and dstack being able to establish + SSH to the returned public address. + +Most likely provider-side causes to confirm on a live rerun: + +1. CloudRift returned a public `host_address` before TCP/22 was actually reachable. +2. The CloudRift host address/NAT/firewall did not expose SSH for the VM. +3. Guest `sshd` or cloud-init did not complete even though CloudRift reported VM + `ready: true`. +4. `host_address` was shared or stale and not mapped to the VM's SSH service. + +Evidence for provider/network exposure being the leading hypothesis: + +- Three separate CloudRift runs returned the same public host address `211.21.50.85`. +- The runs had different internal host addresses: + `10.21.106.174`, `10.21.106.75`, and `10.21.106.99`. +- All timed out on TCP/22 from the server host. +- No image pull or job logs were emitted, so the workload never reached container + startup. +- Native CloudRift API later showed the first two instance IDs as `Inactive`, with the + same `host_address`; the latest repro moved to `Deactivating` immediately after stop. + None allowed post-mortem SSH because TCP/22 never became reachable. + +Ready-to-send issue summary: + +```text +Title: CloudRift VM reports ready but dstack server cannot reach returned host_address:22 + +Three dstack runs pinned to CloudRift ap-east-tw-kn-1 / rtx49-10c-kn.1 stayed in provisioning +before image pull/logs. CloudRift API reported VM mode VirtualMachine and VM ready/running, +and dstack received host_address 211.21.50.85 with ssh_port 22 and username riftuser. +From the same host running dstack server, TCP connect to 211.21.50.85:22 timed out and SSH +to riftuser@211.21.50.85 timed out. Two separate minimal nvidia-smi task runs reproduced +the same behavior. Please check whether host_address 211.21.50.85 should expose SSH for these VMs, +whether VM ready can be true before SSH/NAT/firewall is ready, and whether this host address +mapping is stale/shared. + +Affected instance ids: +- 59aa1b06-77ac-11f1-9b61-937d98c38d58 +- d8d2366a-77ad-11f1-9535-cb9f4e143a57 +- 70bb964a-77b0-11f1-bc41-ab4db0e59660 + +Observed host address: 211.21.50.85 +Observed internal host addresses: 10.21.106.174, 10.21.106.75, 10.21.106.99 +``` + +Harness findings: + +- The endpoint agent used a blocking shell wait loop: + + ```bash + until s=$(dstack run get qwen-endpoint-happy-serving --json ...); do + echo "status=$s ...waiting" + sleep 15 + done + ``` + +- This hid intermediate reasoning until the run was externally stopped. +- The agent recorded the planned offer as RunPod A5000 `$0.27`, but the actual + provisioned offer was CloudRift RTX4090 `$0.39`. Learned presets must use actual + `job_provisioning_data`, not the agent's planned candidate note. + +Independent reproduction: + +Run a tiny non-endpoint task constrained to the same CloudRift region and instance type, +then test whether it also gets stuck before image pull. + +Reproduction config: + +```yaml +type: task +name: cloudrift-rtx49-provisioning-smoke + +image: nvidia/cuda:12.4.1-base-ubuntu22.04 +commands: + - nvidia-smi + +resources: + gpu: 24GB..:1 + disk: 30GB.. + +backends: [cloudrift] +regions: [ap-east-tw-kn-1] +instance_types: [rtx49-10c-kn.1] +spot_policy: on-demand +max_price: 0.5 +max_duration: 15m +``` + +Reproduction run: + +- Run name: `cloudrift-rtx49-provisioning-smoke` +- Config file: `cloudrift-rtx49-provisioning-smoke.dstack.yml` +- Submitted: `2026-07-04 15:39:55` +- Manually stopped: `2026-07-04 15:41:28` +- Cost when stopped: `$0.0108` +- Result: Reproduced. + +Reproduction dstack events: + +- `2026-07-04 15:39:55` run submitted. +- `2026-07-04 15:39:55` job created. +- `2026-07-04 15:39:55` instance `quick-duck-0` created, status `PENDING`. +- `2026-07-04 15:39:58` job moved `SUBMITTED -> PROVISIONING`. +- `2026-07-04 15:39:58` instance moved to `PROVISIONING`. +- `2026-07-04 15:40:04` run moved `SUBMITTED -> PROVISIONING`. +- No image pull progress appeared. +- No task logs appeared. + +Reproduction dstack provisioning data: + +- Backend: `cloudrift` +- Region: `ap-east-tw-kn-1` +- Instance id: `d8d2366a-77ad-11f1-9535-cb9f4e143a57` +- Instance type: `rtx49-10c-kn.1` +- GPU: `RTX4090:24GB:1` +- Host address reported to dstack: `211.21.50.85` +- Price: `$0.39/hr` + +Reproduction native CloudRift API observation: + +- CloudRift status: `Active` +- Node status: `Ready` +- Node mode: `VirtualMachine` +- Host address: `211.21.50.85` +- Internal host address: `10.21.106.75` +- VM `ready`: `true` +- VM state: `Running` +- VM name: `ubuntu-jammy-server-gpu-20250904-011801-1783172397` + +Reproduction network observation from the server host: + +```text +tcp_connect_failed timeout timed out elapsed 8.01 +``` + +Reproduction SSH observation: + +```text +ssh: connect to host 211.21.50.85 port 22: Operation timed out +``` + +Reproduction criteria used: + +- Reproduced if the task also stays in `provisioning` with no image pull/logs while + CloudRift reports VM `Active`, `ready: true`, and TCP/22 times out from the server host. +- Not reproduced if the task pulls the image, runs `nvidia-smi`, and exits. +- If SSH becomes reachable, collect `dstack-shim`, cloud-init, Docker, and `nvidia-smi` + logs before stopping/deleting the instance. + +Next harness changes suggested by this incident: + +- For backend provisioning waits, use short status probes and write `agent_state.json` + after each probe. +- After several unchanged provisioning polls with no image pull/logs, inspect dstack + events and native backend state. +- If native backend says the VM is ready but SSH/TCP is unreachable, mark candidate as a + backend provisioning issue and try a different offer/backend. +- Add a candidate result record that distinguishes planned offer from actual + `job_provisioning_data`. + +### EP-HARNESS-2026-07-04-002: Agent recovered with RunPod but failed to return a report + +Status: Reproduced during endpoint happy-path test. + +Context: + +- Endpoint name: `qwen-endpoint-happy` +- First candidate: CloudRift `rtx49-10c-kn.1`, stopped after stuck provisioning. +- Second candidate: RunPod A5000 in `CA-MTL-1`, `$0.27/hr`. + +What happened: + +- After the CloudRift candidate was externally stopped, the agent resumed. +- It grouped offers by backend and chose RunPod A5000 as the next candidate. +- It edited `service.dstack.yml` to constrain the service to RunPod. +- It submitted `qwen-endpoint-happy-serving` again. +- The new RunPod run reached `RUNNING`. +- vLLM started successfully and loaded `Qwen/Qwen3-0.6B`. +- dstack HTTP probes reached `/v1/chat/completions` and returned `200 OK`. +- The endpoint still failed because the server agent process exited before returning a + final verification report. + +RunPod actual provisioning data: + +- Backend: `runpod` +- Region: `CA-MTL-1` +- Instance type: `NVIDIA RTX A5000` +- GPU: `A5000:24GB:1` +- CPU: `9` +- Memory: `51200 MiB` +- Disk: `30 GiB` +- Price: `$0.27/hr` +- Host: `69.30.85.207` +- SSH port: `22198` + +Service evidence: + +- `dstack run get qwen-endpoint-happy-serving --json` reported run status `running`. +- dstack probe success streak reached `5`. +- vLLM logs included: + + ```text + Starting vLLM server on http://0.0.0.0:8000 + Application startup complete. + 127.0.0.1:44128 - "POST /v1/chat/completions HTTP/1.1" 200 OK + ``` + +Direct verification over SSH: + +```text +/v1/models returned model id Qwen/Qwen3-0.6B. +/v1/chat/completions returned HTTP 200 with model Qwen/Qwen3-0.6B. +``` + +Proxy verification issue: + +- Requests through `http://127.0.0.1:3000/proxy/services/main/qwen-endpoint-happy-serving/...` + initially returned `404 Service main/qwen-endpoint-happy-serving not found` before the + job was marked registered. +- After `JobModel.registered` became true, the same proxy path returned `403 + Unauthenticated or unauthorized to access project main` when using the local CLI + project token. +- Direct SSH verification proved the service itself was healthy, so the agent should not + have failed without a final report. + +Harness findings: + +- The agent should not rely on only the public proxy path for final verification. +- If the proxy returns auth/registration errors but dstack probes are passing, the agent + should verify through `dstack attach`/SSH or another dstack-supported direct path. +- The agent must always return a structured final report on terminal success or failure. +- A successful final service should save a preset using actual RunPod provisioning data, + not planned candidate notes. + +### EP-BACKEND-2026-07-04-003: RunPod A5000 offer listed, but pinned smoke task failed with no capacity + +Status: Reproduced with a separate minimal task. + +Context: + +- Purpose: development preflight before judging endpoint agent behavior. +- Backend: `runpod` +- Region: `CA-MTL-1` +- Instance type: `NVIDIA RTX A5000` +- Price shown by offer/apply preview: `$0.27/hr` +- Fleet template: `endpoint-agent-runpod-a5000-template` +- Fleet template nodes: `0..1` + +Important distinction: + +- RunPod is a container-based backend and cannot be pre-provisioned. The fleet with + `nodes: 0..1` is only a pinned template. The actual capacity test is the smoke task. + +Reproduction config: + +```yaml +type: task +name: endpoint-agent-runpod-a5000-smoke + +image: nvidia/cuda:12.4.1-base-ubuntu22.04 +commands: + - nvidia-smi + +resources: + gpu: 24GB..:1 + disk: 30GB.. + +fleets: [endpoint-agent-runpod-a5000-template] +spot_policy: on-demand +max_price: 0.3 +max_duration: 15m +``` + +Offer/apply preview: + +- `dstack offer --backend runpod --gpu 24GB.. --on-demand --max-price 0.5 --region CA-MTL-1` + still listed `NVIDIA RTX A5000` at `$0.27/hr`. +- `dstack apply` preview for the smoke task selected the same offer through the pinned + fleet template. + +Run result: + +- Submitted: `2026-07-04 15:51:13` +- Run id: `6b0a14e5-270e-45bd-8fe2-1e7524a83427` +- Job submission id: `dd5ec06e-442f-4cc3-9601-e7d60ef513ac` +- Status: `failed` +- Job status message: `no offers` +- Termination reason: `failed_to_start_due_to_no_capacity` +- Cost: `$0.0` +- `job_provisioning_data`: `null` +- No image pull progress. +- No logs. + +Events: + +```text +[2026-07-04 15:51:13] [run endpoint-agent-runpod-a5000-smoke] Run submitted. Status: SUBMITTED +[2026-07-04 15:51:13] [job endpoint-agent-runpod-a5000-smoke-0-0] Job created on run submission. Status: SUBMITTED +[2026-07-04 15:51:13] [instance endpoint-agent-runpod-a5000-template-0, job endpoint-agent-runpod-a5000-smoke-0-0] Instance created for job. Instance status: PENDING +[2026-07-04 15:51:14] [job endpoint-agent-runpod-a5000-smoke-0-0] Job status changed SUBMITTED -> TERMINATING. Termination reason: FAILED_TO_START_DUE_TO_NO_CAPACITY +``` + +What this rules out: + +- This was not a model, vLLM, image, or SSH issue: the run failed before provisioning data, + image pull, or logs. +- This did not spend GPU money. + +Working hypothesis: + +The RunPod offer list/apply preview can show a CA-MTL-1 A5000 offer that is not actually +allocatable at submission time. For agent testing, avoid treating a listed offer as proven +capacity until a smoke task has reached provisioning/logs/running. diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md new file mode 100644 index 0000000000..516909b54e --- /dev/null +++ b/endpoint-agent-checkpoints.md @@ -0,0 +1,115 @@ +# Endpoint Agent Checkpoints + +This file tracks local endpoint-agent checkpoints: what code version was tested, what +actually worked, how much it cost, and what to improve next. It is intentionally local +and practical, not product documentation. + +## Recovery Workflow + +Use this branch for experimental endpoint-agent work: + +```bash +git switch endpoint-agent-checkpoints +``` + +Each useful checkpoint should have: + +- a git commit +- a local tag named `endpoint-agent/` +- the exact model, endpoint, service run, backend, hardware, and cost +- the focused tests/linters that passed +- links or paths to useful local runtime artifacts +- the main known issues before the next checkpoint + +To return to a known-good checkpoint: + +```bash +git switch endpoint-agent-checkpoints +git checkout endpoint-agent/ +``` + +For normal development after inspecting an old checkpoint: + +```bash +git switch endpoint-agent-checkpoints +``` + +## Checkpoint: qwen-runpod-v0-running + +Status: known-good first real endpoint-agent smoke. + +Expected tag after commit: + +```bash +endpoint-agent/qwen-runpod-v0-running +``` + +Date: 2026-07-04 +Base branch tip before this work: `be48b560e` (`Fix TestGetRunsTable::test_simple_run (#3999)`) + +### What Worked + +- Endpoint `qwen-endpoint-happy` reached `running`. +- Model: `Qwen/Qwen3-0.6B`. +- Agent submitted and verified service run `qwen-happy-v2`. +- Final run ID: `b0831bdc-7456-42ab-8c43-ee250427717f`. +- Endpoint URL: `/proxy/services/main/qwen-happy-v2/v1`. +- Backend/hardware: RunPod `CA-MTL-1`, NVIDIA A40 48GB, 9 CPU, 50GB RAM, 60GB disk. +- Hourly price: `$0.44/hr`. +- Reported run cost at checkpoint time: `$0.0599`. +- Service probe had `success_streak: 20`. +- `dstack logs qwen-endpoint-happy --since 1m` resolved through the endpoint name and showed HTTP 200 model requests. +- Preset was saved at: + `/Users/dstack/.dstack/server/data/endpoint_presets/qwen-qwen3-0-6b-b0831bdc.dstack.yml`. +- Saved preset captured the service YAML plus replica resource evidence for replica group `0`. + +### Verification Commands + +```bash +uv run dstack endpoint get qwen-endpoint-happy --json +uv run dstack run get qwen-happy-v2 --json +uv run dstack logs qwen-endpoint-happy --since 1m +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/cli/commands/logs.py src/tests/_internal/cli/commands/test_logs.py src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +uv run ruff check $(git diff --cached --name-only -- '*.py') +``` + +Results observed on 2026-07-04: + +- focused pytest: `17 passed` +- broader endpoint pytest: `110 passed, 42 skipped` +- focused ruff: `All checks passed!` +- staged Python ruff: `All checks passed!` +- endpoint status: `running` +- service run status: `running` + +### Useful Runtime Artifacts + +These are outside the repo and are not part of the commit: + +- Final report: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/0a17fa5a-da2e-4af7-be12-aeaf6666fc3b/workspace/final_report.json` +- Verification: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/0a17fa5a-da2e-4af7-be12-aeaf6666fc3b/workspace/verification.json` +- Saved preset: + `/Users/dstack/.dstack/server/data/endpoint_presets/qwen-qwen3-0-6b-b0831bdc.dstack.yml` + +### Known Issues / Next Hardening + +- The endpoint service `qwen-happy-v2` may still be running and spending `$0.44/hr`. +- Server logs can still become noisy when agent output contains large YAML/CLI output. +- Endpoint logs now resolve to the backing service, but debug trace storage and readable + endpoint-agent logs need more real-run pressure. +- Agent guidance improved after the first run, but the harness still needs repeated + real examples before we can trust the abstractions. +- CloudRift was excluded from endpoint testing after a separate provisioning failure; + details are in `endpoint-agent-backend-troubleshooting.md`. +- The current smoke is a single-replica vLLM deployment. Later checkpoints need harder + models, different hardware, no-offer fallback behavior, and failed-candidate cleanup. + +### Scope Notes + +This checkpoint should save the code version and the first working evidence. It should +not be treated as v1 quality. The point is to keep a recoverable version while the real +agent loop evolves through repeated tests. diff --git a/endpoint-agent-harness-test-plan.md b/endpoint-agent-harness-test-plan.md new file mode 100644 index 0000000000..e11e21c613 --- /dev/null +++ b/endpoint-agent-harness-test-plan.md @@ -0,0 +1,729 @@ +# Endpoint Agent Harness Test Plan + +This document is for testing the agent harness, not for selling the feature. + +The core question is not "can Claude write a service YAML?" The core question is: + +> Can the endpoint agent efficiently converge on a verified dstack service for a model, under real +> project capacity and budget constraints, while leaving enough evidence to improve the harness after +> every failure? + +If a test does not answer that question, it is not a useful harness test. + +## Current Research Baseline + +### Agent runtime facts to rely on + +- Claude Code print/headless mode supports `--output-format stream-json`, `--json-schema`, `--max-turns`, and `--max-budget-usd`. These are useful for subprocess integration, structured final reports, and API-spend caps. +- `--max-budget-usd` caps Claude API spend only. It does not cap GPU spend from dstack runs. The harness must separately track candidate run lifetimes and stop/abort GPU candidates. +- Claude Code plugins/skills are useful for packaging context, but they are not the harness. A plugin can expose skills; it does not give us state, candidate accounting, cleanup, resume, spend tracking, or verification gates. +- Public harness engineering writeups emphasize the same lesson: harness design changes outcomes materially, and useful harnesses rely on traces, self-verification/evaluator separation, context handoff artifacts, and controlled execution loops. + +Primary references: + +- https://docs.anthropic.com/en/docs/claude-code/cli-reference +- https://code.claude.com/docs/en/headless +- https://code.claude.com/docs/en/plugins-reference +- https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents +- https://www.anthropic.com/engineering/harness-design-long-running-apps +- https://www.langchain.com/blog/improving-deep-agents-with-harness-engineering +- https://www.langchain.com/blog/the-anatomy-of-an-agent-harness + +### Deployment recipe sources to rely on + +- vLLM has a model recipe index at `https://recipes.vllm.ai/models.json`; as of this check it returned 263 entries and per-model JSON links. +- SGLang exposes `https://docs.sglang.ai/llms.txt`, including cookbook pages for Qwen3, GLM, Llama, and other families, plus advanced pages such as EPD disaggregation. +- `Qwen/Qwen3-0.6B` has a Hugging Face model card with direct vLLM and SGLang launch examples and OpenAI-compatible curl examples. This makes it a good first real model, because the agent should not need to invent the serving path. +- Advanced writeups such as the LMSYS agent-assisted SGLang development post and Wafer GLM-on-AMD post are not v1 requirements, but they define the direction of the harness: model/framework/hardware-specific experimentation, not a generic YAML generator. + +Primary references: + +- https://recipes.vllm.ai/models.json +- https://docs.vllm.ai/ +- https://docs.sglang.ai/llms.txt +- https://docs.sglang.ai/cookbook/autoregressive/Qwen/Qwen3.md +- https://huggingface.co/Qwen/Qwen3-0.6B +- https://qwen.readthedocs.io/en/latest/deployment/vllm.html +- https://qwen.readthedocs.io/en/latest/deployment/sglang.html +- https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development/ +- https://www.wafer.ai/blog/glm52-amd + +### Product / benchmark references to learn from + +These references should influence harness shape and later evaluation, not v1 feature scope. + +- Modal Auto Endpoints frames the product as "one command to a model endpoint" while keeping the generated app/code, GPU selection, regionalization, engine flags, metrics, and benchmark results inspectable. The v1 dstack endpoint agent should copy the inspectability principle, not Modal-specific infrastructure or load tuning. +- Makora frames the frontier as full-stack optimization: orchestration, routing/scheduling, engine tuning, speculative decoding, quantization, kernels, and heterogeneous hardware. This is a useful long-term direction for the agent harness, but most of it is explicitly out of v1. +- Runpod Overdrive's benchmark writeup is useful because it evaluates model serving by workload profile, not just model name. It compares configurations across chatbot, RAG, code-generation, and long-form generation workloads and reports throughput, ITL, TTFT, end-to-end latency, quality, and cost/token. This is Later for endpoint optimization, but v1 artifacts should not make it impossible to add these measurements. + +Additional references: + +- https://modal.com/blog/introducing-auto-endpoints +- https://www.makora.com/ +- https://www.runpod.io/blog/overdrive-benchmarks + +## What We Are Actually Testing + +The harness has five jobs: + +1. Give the agent the right operating context. +2. Bound and observe experiments. +3. Separate candidate execution from final endpoint activation. +4. Preserve evidence for repeatability and improvement. +5. Fail usefully instead of looping or spending blindly. + +The v0 loop should be intentionally simple, but it must already exercise these jobs. + +## Development Preflight: Separate Backend From Agent + +During development, do not use the endpoint agent as the first proof that a backend, +region, instance type, image, SSH path, or fleet/run provisioning path works. Before +asking the agent to spend serious time on a target hardware path, run a small independent +dstack preflight with the same constraints. + +This preflight is a development/testing practice, not a production endpoint requirement. +Normal endpoint usage should still allow the agent and/or preset path to provision +through dstack. + +Recommended preflight order: + +1. Check offers with the same backend/region/instance/max-price/spot constraints. +2. For VM-based backends, optionally create or update a fleet separately and wait until + it can pre-provision or reuse capacity. +3. For container-based backends such as RunPod and Vast.ai, do not treat fleet creation + as pre-provisioning. Use a pinned `nodes: 0..1` fleet template or direct run + constraints, then submit a tiny detached task that forces real provisioning. +4. Submit the tiny task on the target hardware, for example `nvidia-smi` in a CUDA base + image, with a short `max_duration`. +5. Confirm the run reaches image pull/logs/running or records a concrete provisioning + failure. +6. If it fails before image pull/logs, diagnose as a dstack/backend issue first: events, + run JSON, fleet state, native backend state, TCP/SSH, shim/cloud-init where reachable. +7. Record confirmed backend/dstack issues in `endpoint-agent-backend-troubleshooting.md` + with a minimal reproduction. + +Only after this preflight passes should a development run treat agent behavior as the +primary thing under test. This prevents mixing "agent made a poor decision" with "the +selected backend cannot currently provision reachable capacity." + +## Harness Components Under Test + +### Supervisor + +The supervisor is server-side code or a standalone prototype runner that starts the agent process, passes endpoint constraints, streams/traces output, and enforces hard gates. + +Required responsibilities: + +- create isolated workspace and HOME +- create scoped dstack CLI config +- pass only required env vars +- start Claude Code with structured final report schema +- stream compact endpoint logs +- write full debug trace when debug is enabled +- track candidate run names/ids observed from agent output or final report +- stop active candidates on abort/failure when safe +- reject final success without a verified final service report + +### Agent + +The agent is allowed to: + +- read/write files in the workspace +- use web search/fetch +- use shell commands +- invoke the real `dstack` CLI +- create service/task/dev-environment YAMLs for experiments +- submit detached dstack runs within endpoint constraints +- stop bad candidates + +The agent is not allowed to: + +- use hidden server APIs +- bypass endpoint profile constraints +- mark the endpoint running without a real model request +- keep multiple GPU candidates running without an explicit reason +- write secrets to YAML, logs, presets, or final reports + +### Deterministic Observer + +The observer is not an LLM. It reads dstack state, events, logs, and workspace artifacts to decide whether the run satisfied gates. + +Required checks: + +- every submitted run had a recorded candidate entry +- no active non-final candidate remains after terminal failure +- final run exists and is the required service run +- final service has `model` and model URL +- final verification request is recorded +- endpoint constraints were not violated in previews/submissions +- final preset, if saved, includes final service YAML and replica resource evidence + +## Required Workspace Artifacts + +The agent workspace must contain these files by the end of every attempt. They can be written by the agent, the supervisor, or both, but the deterministic observer must be able to parse them. + +### `agent_state.json` + +Purpose: current phase and budget/candidate bookkeeping. + +Required fields: + +```json +{ + "endpoint_name": "qwen-endpoint-agent-smoke", + "model": "Qwen/Qwen3-0.6B", + "phase": "research|capacity|experiment|verify|success|failure", + "max_agent_budget": 2.0, + "max_hourly_price": 0.3, + "started_at": "ISO-8601", + "updated_at": "ISO-8601" +} +``` + +### `sources.jsonl` + +Purpose: recipe and hardware grounding. + +One JSON object per source: + +```json +{ + "url": "https://huggingface.co/Qwen/Qwen3-0.6B", + "kind": "model-card|framework-doc|recipe|dstack-doc|deployment-report|log-evidence", + "claim": "HF card provides vLLM and SGLang launch examples for this model", + "used_for": "serving command selection", + "confidence": "high|medium|low" +} +``` + +### `hardware_reasoning.md` + +Purpose: make the hardware decision reviewable. + +Must include: + +- model size and serving mode +- expected framework +- expected VRAM and disk class +- selected dstack constraints +- why the selected offer/fleet is credible +- why cheaper/other offers were rejected if applicable + +### `candidates.jsonl` + +Purpose: every spend-capable experiment. + +One JSON object per candidate transition: + +```json +{ + "candidate": "qwen-endpoint-agent-smoke-serving", + "kind": "service|task|dev-environment", + "role": "prototype|final", + "run_id": null, + "status": "planned|previewed|submitted|provisioning|running|verified|rejected|stopping|stopped|failed", + "hourly_price": 0.24, + "resources": "gpu=RTX2000Ada:16GB disk=100GB", + "reason": "first final service candidate from HF vLLM recipe", + "timestamp": "ISO-8601" +} +``` + +### `commands.jsonl` + +Purpose: audit and loop analysis. + +One JSON object per command: + +```json +{ + "timestamp": "ISO-8601", + "command": "printf 'n\\n' | dstack apply -f service.dstack.yml --max-price 0.3 --on-demand", + "exit_code": 0, + "category": "research|offer|preview|submit|status|events|logs|stop|verify", + "output_path": "command-output/0004.txt" +} +``` + +### `verification.json` + +Purpose: activation gate. + +Required on success: + +```json +{ + "run_name": "qwen-endpoint-agent-smoke-serving", + "model_url": "http://...", + "request_kind": "openai-chat-completions|openai-responses|custom", + "request_model": "Qwen/Qwen3-0.6B", + "status_code": 200, + "response_excerpt": "The capital of France is Paris.", + "verified_at": "ISO-8601" +} +``` + +### `final_report.json` + +Purpose: handoff to endpoint worker and preset saving. + +This is the existing structured report, but the harness tests should reject shallow reports that do not reference the artifact evidence. + +Required on success: + +- `success: true` +- final `run_name` or `run_id` +- final `service_yaml` +- `recipe_sources` +- `verification_summary` + +Required on failure: + +- `success: false` +- `failure_summary` +- link to the last useful evidence: candidate, events, logs, source mismatch, or budget/constraint gate + +## Hard Gates + +These gates are deliberately strict because they protect money and prevent false RUNNING endpoints. + +### Before Any Paid Run + +- Endpoint constraints are parsed and rendered. +- Agent has produced at least one source or explicit reason why no external source is needed. +- Hardware reasoning exists. +- `dstack apply` preview was run with the endpoint constraints. +- Preview output does not violate `max_price`, `spot_policy`, backend, region, fleet, instance type, or reuse constraints. +- The user-confirmed hourly budget is still valid. +- For development live tests, the target backend/hardware path passed an independent + dstack preflight, or the test is explicitly about provisioning-stall recovery. + +### While a Candidate Is Running + +- At most one GPU candidate is active unless the supervisor has an explicit multi-candidate allowance. +- If provisioning has no meaningful progress after the configured observation window, inspect events before continuing. +- If there are no service logs, inspect events and run JSON before resubmitting. +- If a run is stopped externally or `termination_reason` is `stopped_by_user`, do not resubmit automatically. +- Do not repeat the same YAML/offer/image after a failure without a recorded new hypothesis. + +### Before Success + +- Final run is the required service run name. +- Final run is a service, not a task or dev environment. +- Final service exposes a model URL. +- Final model request succeeded. +- The request used the requested model name. +- Final YAML is present and does not include secret values. +- Candidate history shows no active leftover GPU runs. + +### On Failure + +- Stop active endpoint-created GPU candidates when safe. +- Do not save a preset. +- Keep workspace artifacts. +- Failure message in endpoint status must be short. +- Detailed evidence must be in endpoint logs/debug trace/workspace, not stuffed into status events. + +## Metrics + +Every real run should end with these numbers: + +| metric | why it matters | +|---|---| +| time to first valid preview | source + config efficiency | +| time to first paid candidate | whether research is stuck | +| time to first logs | infra vs app debugging | +| time to verified model response | actual endpoint value | +| number of paid candidates | spend efficiency | +| total candidate run minutes | GPU spend exposure | +| agent API spend | Claude budget effectiveness | +| repeated command ratio | loop detection | +| repeated failure ratio | harness quality | +| number of source URLs used | grounding | +| number of source URLs that affected final YAML | real grounding vs noise | +| cleanup completeness | leak prevention | + +## Scenario Ladder + +Run these in order. Do not skip ahead just because the previous scenario looked boring. + +### S0: Offline Harness Contract + +Goal: verify artifacts and gates without dstack spend. + +Input: + +```yaml +type: endpoint +name: qwen-endpoint-contract +model: Qwen/Qwen3-0.6B +preset_policy: create +max_agent_budget: 0.25 +spot_policy: on-demand +max_price: 0.3 +``` + +Run mode: + +- Agent may research and write YAML. +- Agent may run `dstack offer`. +- Agent may run `printf 'n\n' | dstack apply ...`. +- Agent must not run `dstack apply -y`. + +Pass: + +- Required artifacts exist. +- Preview command uses constraints. +- Final report is failure/blocked due no-submit mode, not success. + +Fail: + +- Agent submits a run. +- Agent writes shallow artifacts. +- Agent guesses unsupported flags/YAML. + +### S1: Impossible Constraints + +Goal: no-spend failure on capacity/constraint mismatch. + +Input examples: + +```yaml +type: endpoint +name: qwen-endpoint-no-offers +model: Qwen/Qwen3-0.6B +preset_policy: create +max_agent_budget: 0.25 +spot_policy: on-demand +max_price: 0.001 +``` + +Pass: + +- No paid run submitted. +- `sources.jsonl` and `hardware_reasoning.md` explain that model is deployable but constraints block capacity. +- Failure summary is concise. + +Fail: + +- Agent wastes turns trying random YAML. +- Agent suggests violating max price. + +### S2: First Real Happy Path, Tiny Model + +Goal: first actual endpoint created and verified. + +Candidate model: + +- `Qwen/Qwen3-0.6B` + +Why this model: + +- It is small enough for low-cost GPUs. +- Its model card includes direct vLLM and SGLang launch examples. +- The correct path should be easy enough that failures expose harness issues rather than model complexity. + +Initial config: + +```yaml +type: endpoint +name: qwen-endpoint-agent-smoke +model: Qwen/Qwen3-0.6B +preset_policy: create +max_agent_budget: 2.0 +spot_policy: on-demand +max_price: 0.3 +``` + +Current local capacity observation on 2026-07-04: + +- `dstack offer --max-price 0.3 --on-demand --gpu 1 --max-offers 20` showed multiple offers, mostly Vast.ai plus one RunPod RTX 2000 Ada. +- The previous Vast.ai RTX 5060 Ti attempt stuck in provisioning without service logs. For the next real run, prefer a more stable/common path even if not the cheapest, and ask before increasing `max_price`. +- The CloudRift RTX4090 path reproduced a backend reachability problem with a separate + `nvidia-smi` task; do not use that path for judging agent quality until the backend + issue is understood or resolved. + +Expected agent behavior: + +- Use HF/Qwen source first. +- Prefer vLLM first unless current evidence suggests SGLang is more reliable for the offer/image. +- Use a common CUDA image or Python install path that matches the chosen framework. +- Submit one final service candidate. +- Poll run JSON, events, and logs. +- Verify via OpenAI-compatible chat completion. +- Save preset after success. + +Pass: + +- Verified model response. +- One paid candidate or a clearly justified second candidate. +- Candidate stopped/cleaned if replaced. +- Preset saved with final YAML and observed resources. + +Fail: + +- Agent chooses arbitrary cheapest offer with no reasoning. +- Agent loops on `dstack ps` table parsing. +- Agent resubmits after external stop. +- Agent reports success before model request. + +### S3: Happy Path Replay From Learned Preset + +Goal: prove the preset created by S2 is actually useful without the agent. + +Input: + +```yaml +type: endpoint +name: qwen-endpoint-preset-replay +model: Qwen/Qwen3-0.6B +preset_policy: reuse +spot_policy: on-demand +max_price: 0.3 +``` + +Pass: + +- Plan finds learned preset. +- Plan shows preset and offers. +- Pipeline creates backing service without invoking agent. +- Service reaches running via normal service readiness. + +Fail: + +- Preset lacks enough resource information to replay. +- Preset matching ignores no-offer state. +- Agent is invoked despite `preset_policy: reuse`. + +### S4: Bad Service YAML Recovery + +Goal: test whether the agent can debug its own bad deployment. + +Setup options: + +- Seed prompt/harness with an intentionally bad first hypothesis, such as wrong port or missing `model`. +- Or choose a framework/image combination likely to fail import. + +Pass: + +- Agent notices validation/runtime failure from preview/logs. +- Agent records new hypothesis. +- Agent stops/replaces bad candidate. +- Final candidate is verified. + +Fail: + +- Repeats same apply. +- Treats validation failure as no-offers. +- Leaves failed GPU run active. + +### S5: Provisioning Stall Recovery + +Goal: reproduce the previous stuck-provisioning failure and ensure the agent does not loop. + +Setup: + +- Use a backend/offer class known to have a chance of provisioning stalls, or simulate via fake agent/runtime in tests. + +Pass: + +- Agent polls JSON, then events. +- After bounded no-progress window, agent stops or fails with evidence. +- No resubmission after external stop. + +Fail: + +- Keeps polling table output. +- Dumps huge offer/event output into endpoint status. +- Resubmits after `stopped_by_user`. + +### S6: Env / Gated Model Path + +Goal: verify env handling and secret hygiene. + +Candidate model: + +- A small gated/private HF model if available, or a deliberately missing `HF_TOKEN` scenario. + +Pass: + +- YAML references `HF_TOKEN` by name only. +- Missing/invalid auth is diagnosed from logs. +- Secret value never appears in YAML, endpoint logs, status message, preset, or trace after redaction. + +Fail: + +- Agent writes secret value into YAML. +- Agent treats auth as hardware failure. + +### S7: Medium Model Hardware Reasoning + +Goal: test model/hardware sizing, not just tiny-model happy path. + +Candidate models: + +- `Qwen/Qwen3-4B` +- `Qwen/Qwen3-8B` + +Run only after budget confirmation. + +Expected behavior: + +- Agent estimates VRAM/disk before offers. +- Agent evaluates quantized variants if needed. +- Agent may choose larger GPU over cheapest if reliability/VRAM requires it. + +Pass: + +- Reasoning links model size, precision/quantization, max model length, framework, and selected offer. +- Candidate verifies. + +Fail: + +- Guesses `gpu: 16GB` or `gpu: 80GB` without evidence. +- Confuses parameter size with runtime memory. + +### S8: Framework Choice + +Goal: verify that the harness supports real vLLM vs SGLang choice. + +Input: + +- Use a model with both vLLM and SGLang evidence. + +Pass: + +- Agent records why it chose one framework. +- If first framework fails, switching framework is a new hypothesis with evidence. + +Fail: + +- Agent always uses one framework regardless of source/log evidence. + +### S9: Advanced Scenario, No v1 Implementation + +Goal: ensure advanced sources improve planning without causing premature v1 scope explosion. + +Candidate sources: + +- GLM-on-AMD Wafer post. +- LMSYS agent-assisted SGLang development post. +- SGLang EPD disaggregation docs. + +Pass: + +- Agent identifies advanced deployment patterns and marks them later unless required. +- No multi-service router/worker topology is attempted in v1 by default. + +Fail: + +- Agent tries P/D disaggregation for a simple endpoint. +- Agent ignores advanced evidence for a model that actually needs it. + +## Real Server Path During Development + +Use the actual endpoint worker path for live agent tests. Avoid building a separate +endpoint runner unless there is a specific bug that cannot be isolated through the real +server path, unit tests, workspace artifacts, and the backend preflight above. + +Development live-test order: + +1. Run the independent dstack/backend preflight for the intended hardware path. +2. Apply the endpoint config through `dstack apply -f .dstack.yml`. +3. Watch endpoint logs and the agent workspace artifacts. +4. Stop candidate runs promptly if the agent stalls or the backend preflight evidence no + longer matches reality. +5. If the issue is dstack/backend, reproduce it outside endpoints and report it in + `endpoint-agent-backend-troubleshooting.md`. + +## Server Integration Gates + +Only harden endpoint worker behavior after the real server path or unit tests show it is +needed. Avoid speculative scaffolding. + +### Gate A: Runtime + +- Claude Code subprocess can run non-interactively from server env. +- `--max-budget-usd` is honored for API spend. +- Stream JSON parsing does not block. +- Large tool outputs are kept out of endpoint status. +- Debug trace captures enough detail. + +### Gate B: Candidate Accounting + +- Every paid candidate has a recorded candidate row or artifact. +- Latest/final service run is distinguishable from prototypes. +- Deletion/abort stops active candidates. +- Worker crash recovery can reconcile linked/submitted runs. + +### Gate C: Verification + +- Agent final report alone is not enough. +- Report must reference `verification.json`. +- Server should validate that final run exists and exposes model URL, but the agent owns functional verification. + +### Gate D: Preset Save + +- Preset is saved only after verified success. +- Preset stores final service YAML and ordered replica resource evidence. +- Preset does not include secrets. +- Replay scenario passes. + +## First Real Run Decision + +Next paid run should be S2, but only after confirming: + +- max hourly price +- acceptable backend/provider preference +- whether to avoid Vast.ai for the next attempt after the prior stuck provisioning +- whether to raise max price for a more stable/common NVIDIA offer + +Recommendation: + +- Keep `Qwen/Qwen3-0.6B`. +- Keep `preset_policy: create`. +- Keep `spot_policy: on-demand`. +- Consider preferring RunPod or another stable backend if available under budget, even if Vast.ai is cheaper. +- Do not increase `max_price` without explicit confirmation. + +## What Counts As Harness Improvement + +A change improves the harness only if it moves one of these metrics: + +- fewer paid candidates for same scenario +- lower total GPU minutes +- fewer repeated commands/failures +- better diagnosis on failure +- more reliable cleanup +- better source-to-YAML traceability +- higher replay success from learned preset + +Prompt wording, skills, plugins, or runtime flags are implementation details. They are only useful if the scenario evidence improves. + +## Now vs Later From Product References + +### Now + +- Keep the "one endpoint config to verified service" UX. +- Make the generated service YAML inspectable and save it as preset provenance. +- Record the engine/framework choice and important serving flags in `sources.jsonl` / `hardware_reasoning.md`. +- Record enough final-run metadata to reproduce: model, framework, image/install path, command, resources, observed replica resources, and verification request. +- Treat "no hidden black box" as a v1 principle: if the agent made a deployment decision, the workspace artifacts should show why. +- Keep first verification functional: a real model request proves the endpoint works. + +### Later + +- Load testing and workload-profile optimization. +- Benchmarking across chatbot/RAG/code/long-form profiles. +- TTFT, ITL, throughput, end-to-end latency, quality, and cost/token dashboards. +- Autoscaling and multi-replica tuning. +- Speculative decoding, quantization strategy exploration, engine patching, kernel work, and heterogeneous hardware optimization. +- Agent retry policy that uses production traffic metrics to re-tune a running endpoint. +- Curated benchmarked presets or a registry that includes measured workload profiles. + +### Do Not Do In V1 + +- Do not ask the agent to optimize under load. +- Do not gate endpoint RUNNING on benchmark performance. +- Do not add generic performance dashboards before basic deployment/replay works. +- Do not let performance references push us into P/D disaggregation, routers/workers, or custom kernels for simple endpoints. diff --git a/qwen-endpoint-happy.dstack.yml b/qwen-endpoint-happy.dstack.yml new file mode 100644 index 0000000000..dedf38ae67 --- /dev/null +++ b/qwen-endpoint-happy.dstack.yml @@ -0,0 +1,10 @@ +type: endpoint +name: qwen-endpoint-happy + +model: Qwen/Qwen3-0.6B +preset_policy: create +max_agent_budget: 2 + +spot_policy: on-demand +max_price: 0.5 +backends: [runpod] diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py new file mode 100644 index 0000000000..28d58f53b2 --- /dev/null +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -0,0 +1,149 @@ +import argparse +import time + +from rich.live import Live + +from dstack._internal.cli.commands import APIBaseCommand +from dstack._internal.cli.services.completion import EndpointNameCompleter +from dstack._internal.cli.utils.common import ( + LIVE_TABLE_PROVISION_INTERVAL_SECS, + LIVE_TABLE_REFRESH_RATE_PER_SEC, + confirm_ask, + console, +) +from dstack._internal.cli.utils.endpoint import ( + filter_endpoints_for_listing, + get_endpoints_table, + print_endpoints_table, +) +from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.utils.json_utils import pydantic_orjson_dumps_with_indent + + +class EndpointCommand(APIBaseCommand): + NAME = "endpoint" + DESCRIPTION = "Manage endpoints" + + def _register(self): + super()._register() + self._parser.set_defaults(subfunc=self._list) + subparsers = self._parser.add_subparsers(dest="action") + + list_parser = subparsers.add_parser( + "list", help="List endpoints", formatter_class=self._parser.formatter_class + ) + list_parser.set_defaults(subfunc=self._list) + + for parser in [self._parser, list_parser]: + parser.add_argument( + "-a", + "--all", + help=( + "Show all endpoints. By default, it only shows unfinished endpoints " + "and the last finished endpoint. In watch mode, it only shows " + "unfinished endpoints." + ), + action="store_true", + ) + parser.add_argument( + "-w", + "--watch", + help="Update listing in realtime", + action="store_true", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Show more information" + ) + parser.add_argument( + "-n", + "--last", + help="Show only the last N endpoints. Implies --all", + type=int, + default=None, + ) + + delete_parser = subparsers.add_parser( + "delete", + help="Delete endpoints", + formatter_class=self._parser.formatter_class, + ) + delete_parser.add_argument( + "name", + help="The name of the endpoint", + ).completer = EndpointNameCompleter() # type: ignore[attr-defined] + delete_parser.add_argument( + "-y", "--yes", help="Don't ask for confirmation", action="store_true" + ) + delete_parser.set_defaults(subfunc=self._delete) + + get_parser = subparsers.add_parser( + "get", help="Get an endpoint", formatter_class=self._parser.formatter_class + ) + get_parser.add_argument( + "name", + metavar="NAME", + help="The name of the endpoint", + ).completer = EndpointNameCompleter() # type: ignore[attr-defined] + get_parser.add_argument( + "--json", + action="store_true", + required=True, + help="Output in JSON format", + ) + get_parser.set_defaults(subfunc=self._get) + + def _command(self, args: argparse.Namespace): + super()._command(args) + args.subfunc(args) + + def _list(self, args: argparse.Namespace): + endpoints = self._get_endpoints_for_listing(args) + if not args.watch: + print_endpoints_table(endpoints, verbose=args.verbose) + return + + try: + with Live(console=console, refresh_per_second=LIVE_TABLE_REFRESH_RATE_PER_SEC) as live: + while True: + live.update(get_endpoints_table(endpoints, verbose=args.verbose)) + time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS) + endpoints = self._get_endpoints_for_listing(args) + except KeyboardInterrupt: + pass + + def _get_endpoints_for_listing(self, args: argparse.Namespace): + endpoints = self.api.client.endpoints.list(self.api.project) + return filter_endpoints_for_listing( + endpoints, + show_all=args.all, + limit=args.last, + include_latest_finished=not args.watch, + ) + + def _delete(self, args: argparse.Namespace): + try: + self.api.client.endpoints.get(project_name=self.api.project, name=args.name) + except ResourceNotExistsError: + console.print(f"Endpoint [code]{args.name}[/] does not exist") + exit(1) + + if not args.yes and not confirm_ask(f"Delete the endpoint [code]{args.name}[/]?"): + console.print("\nExiting...") + return + + with console.status("Deleting endpoint..."): + self.api.client.endpoints.delete(project_name=self.api.project, names=[args.name]) + + console.print(f"Endpoint [code]{args.name}[/] deleted") + + def _get(self, args: argparse.Namespace): + try: + endpoint = self.api.client.endpoints.get( + project_name=self.api.project, + name=args.name, + ) + except ResourceNotExistsError: + console.print("Endpoint not found") + exit(1) + + print(pydantic_orjson_dumps_with_indent(endpoint.dict(), default=None)) diff --git a/src/dstack/_internal/cli/commands/logs.py b/src/dstack/_internal/cli/commands/logs.py index 78cde52f49..3993050e73 100644 --- a/src/dstack/_internal/cli/commands/logs.py +++ b/src/dstack/_internal/cli/commands/logs.py @@ -1,10 +1,14 @@ import argparse +import base64 import sys +from typing import Iterable from dstack._internal.cli.commands import APIBaseCommand -from dstack._internal.cli.services.completion import RunNameCompleter +from dstack._internal.cli.services.completion import RunOrEndpointNameCompleter from dstack._internal.cli.utils.common import get_start_time -from dstack._internal.core.errors import CLIError +from dstack._internal.core.errors import CLIError, ResourceNotExistsError +from dstack._internal.core.models.endpoints import Endpoint +from dstack._internal.server.schemas.logs import PollLogsRequest from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -39,24 +43,82 @@ def _register(self): ), type=str, ) - self._parser.add_argument("run_name").completer = RunNameCompleter(all=True) # type: ignore[attr-defined] + self._parser.add_argument("run_name").completer = RunOrEndpointNameCompleter() # type: ignore[attr-defined] def _command(self, args: argparse.Namespace): super()._command(args) - run = self.api.runs.get(args.run_name) - if run is None: - raise CLIError(f"Run {args.run_name} not found") - start_time = get_start_time(args.since) - logs = run.logs( - start_time=start_time, - diagnose=args.diagnose, - replica_num=args.replica, - job_num=args.job, - ) + logs = self._get_logs(args=args, start_time=start_time) try: for log in logs: sys.stdout.buffer.write(log) sys.stdout.buffer.flush() except KeyboardInterrupt: pass + + def _get_logs( + self, + args: argparse.Namespace, + start_time, + ) -> Iterable[bytes]: + endpoint = self._get_endpoint(args.run_name) + if endpoint is not None: + return self._get_endpoint_logs(endpoint=endpoint, args=args, start_time=start_time) + + run = self.api.runs.get(args.run_name) + if run is not None: + return run.logs( + start_time=start_time, + diagnose=args.diagnose, + replica_num=args.replica, + job_num=args.job, + ) + + raise CLIError(f"Run or endpoint {args.run_name} not found") + + def _get_endpoint(self, name: str) -> Endpoint | None: + try: + return self.api.client.endpoints.get( + project_name=self.api.project, + name=name, + ) + except ResourceNotExistsError: + return None + + def _get_endpoint_logs( + self, + endpoint: Endpoint, + args: argparse.Namespace, + start_time, + ) -> Iterable[bytes]: + if endpoint.run_name is not None: + run = self.api.runs.get(endpoint.run_name) + if run is not None: + yield from run.logs( + start_time=start_time, + diagnose=args.diagnose, + replica_num=args.replica, + job_num=args.job, + ) + return + + next_token = None + while True: + resp = self.api.client.logs.poll( + project_name=self.api.project, + body=PollLogsRequest( + run_name=endpoint.name, + job_submission_id=endpoint.id, + start_time=start_time, + end_time=None, + descending=False, + limit=1000, + diagnose=False, + next_token=next_token, + ), + ) + for log in resp.logs: + yield base64.b64decode(log.message) + next_token = resp.next_token + if next_token is None: + break diff --git a/src/dstack/_internal/cli/main.py b/src/dstack/_internal/cli/main.py index 32f15a95f8..335f7693f1 100644 --- a/src/dstack/_internal/cli/main.py +++ b/src/dstack/_internal/cli/main.py @@ -8,6 +8,7 @@ from dstack._internal.cli.commands.attach import AttachCommand from dstack._internal.cli.commands.completion import CompletionCommand from dstack._internal.cli.commands.delete import DeleteCommand +from dstack._internal.cli.commands.endpoint import EndpointCommand from dstack._internal.cli.commands.event import EventCommand from dstack._internal.cli.commands.export import ExportCommand from dstack._internal.cli.commands.fleet import FleetCommand @@ -67,6 +68,7 @@ def main(): ApplyCommand.register(subparsers) AttachCommand.register(subparsers) DeleteCommand.register(subparsers) + EndpointCommand.register(subparsers) EventCommand.register(subparsers) ExportCommand.register(subparsers) FleetCommand.register(subparsers) diff --git a/src/dstack/_internal/cli/services/completion.py b/src/dstack/_internal/cli/services/completion.py index 4fa276945d..d80a211a33 100644 --- a/src/dstack/_internal/cli/services/completion.py +++ b/src/dstack/_internal/cli/services/completion.py @@ -60,6 +60,13 @@ def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.runs.list(self.all)] +class RunOrEndpointNameCompleter(BaseAPINameCompleter): + def fetch_resource_names(self, api: Client) -> Iterable[str]: + run_names = [r.name for r in api.runs.list(all=True)] + endpoint_names = [r.name for r in api.client.endpoints.list(api.project)] + return sorted(set(run_names + endpoint_names)) + + class FleetNameCompleter(BaseAPINameCompleter): def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.client.fleets.list(api.project)] @@ -70,6 +77,11 @@ def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.client.volumes.list(api.project)] +class EndpointNameCompleter(BaseAPINameCompleter): + def fetch_resource_names(self, api: Client) -> Iterable[str]: + return [r.name for r in api.client.endpoints.list(api.project)] + + class GatewayNameCompleter(BaseAPINameCompleter): def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.client.gateways.list(api.project)] diff --git a/src/dstack/_internal/cli/services/configurators/__init__.py b/src/dstack/_internal/cli/services/configurators/__init__.py index 91768bdcd3..e5cbd0f69f 100644 --- a/src/dstack/_internal/cli/services/configurators/__init__.py +++ b/src/dstack/_internal/cli/services/configurators/__init__.py @@ -5,6 +5,7 @@ import yaml from dstack._internal.cli.services.configurators.base import BaseApplyConfigurator +from dstack._internal.cli.services.configurators.endpoint import EndpointConfigurator from dstack._internal.cli.services.configurators.fleet import FleetConfigurator from dstack._internal.cli.services.configurators.gateway import GatewayConfigurator from dstack._internal.cli.services.configurators.run import ( @@ -32,6 +33,7 @@ DevEnvironmentConfigurator, TaskConfigurator, ServiceConfigurator, + EndpointConfigurator, FleetConfigurator, GatewayConfigurator, VolumeConfigurator, diff --git a/src/dstack/_internal/cli/services/configurators/endpoint.py b/src/dstack/_internal/cli/services/configurators/endpoint.py new file mode 100644 index 0000000000..807f291c7e --- /dev/null +++ b/src/dstack/_internal/cli/services/configurators/endpoint.py @@ -0,0 +1,431 @@ +import argparse +import shutil +import time +from pathlib import Path + +from rich.table import Table + +from dstack._internal.cli.services.configurators.base import ( + ApplyEnvVarsConfiguratorMixin, + BaseApplyConfigurator, +) +from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args +from dstack._internal.cli.utils.common import ( + NO_OFFERS_WARNING, + confirm_ask, + console, + format_backend, + format_instance_availability, +) +from dstack._internal.cli.utils.endpoint import get_endpoints_table +from dstack._internal.cli.utils.rich import MultiItemStatus +from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.core.models.configurations import ApplyConfigurationType +from dstack._internal.core.models.endpoints import ( + AnyEndpointProvisioningPlan, + Endpoint, + EndpointConfiguration, + EndpointPlan, + EndpointPresetPolicy, + EndpointProvisioningPlanAgent, + EndpointProvisioningPlanNone, + EndpointProvisioningPlanPreset, + EndpointStatus, +) +from dstack._internal.core.models.profiles import Profile, ProfileParams, SpotPolicy +from dstack._internal.utils.common import local_time, make_proxy_url +from dstack.api.utils import load_profile + +_NO_ENDPOINT_FLEETS_WARNING = ( + "[error]" + "The project has no fleets. Create one before submitting an endpoint.\n" + "See [link]https://dstack.ai/docs/guides/troubleshooting/#no-fleets[/link]" + "[/]\n" +) + + +class EndpointConfigurator( + ApplyEnvVarsConfiguratorMixin, + BaseApplyConfigurator[EndpointConfiguration], +): + TYPE = ApplyConfigurationType.ENDPOINT + + def apply_configuration( + self, + conf: EndpointConfiguration, + configuration_path: str, + command_args: argparse.Namespace, + configurator_args: argparse.Namespace, + ): + self.apply_args(conf, configurator_args) + with console.status("Getting apply plan..."): + plan = self.api.client.endpoints.get_plan( + project_name=self.api.project, + configuration=conf, + configuration_path=configuration_path, + ) + no_fleets = False + if _preset_plan_has_no_offers(plan.provisioning_plan): + if len(self.api.client.fleets.list(self.api.project, include_imported=True)) == 0: + no_fleets = True + _print_endpoint_plan(plan, no_fleets=no_fleets) + + confirm_message = _get_apply_confirm_message(plan) + current_resource = _get_non_terminal_current_resource(plan) + stop_run_name = None + if current_resource is not None: + if current_resource.configuration == plan.configuration: + console.print( + f"Endpoint [code]{current_resource.name}[/] already exists." + " Detected no changes." + ) + if command_args.yes and not command_args.force: + console.print("Use --force to apply anyway.") + return + else: + # TODO: Replace v1 stop/delete/recreate with endpoint in-place update + # via service rolling deployment once endpoint versioning exists. + console.print( + f"Endpoint [code]{current_resource.name}[/] already exists." + " Detected changes that [error]cannot[/] be updated in-place." + ) + else: + serving_run_name = _get_endpoint_serving_run_name(plan) + same_name_run = None + if serving_run_name is not None: + same_name_run = self.api.runs.get(serving_run_name) + if same_name_run is not None and not same_name_run.status.is_finished(): + stop_run_name = same_name_run.name + console.print( + f"Active run [code]{stop_run_name}[/] already exists." + " The endpoint will use this backing service run name." + ) + confirm_message = _get_apply_confirm_message( + plan, + stop_run_name=stop_run_name, + ) + + if not command_args.yes and not confirm_ask(confirm_message): + console.print("\nExiting...") + return + + if current_resource is not None: + with console.status("Deleting existing endpoint..."): + self.api.client.endpoints.delete( + project_name=self.api.project, + names=[current_resource.name], + ) + while True: + try: + self.api.client.endpoints.get( + project_name=self.api.project, + name=current_resource.name, + ) + except ResourceNotExistsError: + break + time.sleep(1) + + if stop_run_name is not None: + with console.status("Stopping run..."): + self.api.client.runs.stop(self.api.project, [stop_run_name], abort=False) + while True: + run = self.api.runs.get(stop_run_name) + if run is None or run.status.is_finished(): + break + time.sleep(1) + + with console.status("Creating endpoint..."): + endpoint = self.api.client.endpoints.create( + project_name=self.api.project, + configuration=conf, + ) + if command_args.detach: + _print_submitted_endpoint_message(endpoint) + return + self._follow_endpoint_apply(endpoint) + + def delete_configuration( + self, + conf: EndpointConfiguration, + configuration_path: str, + command_args: argparse.Namespace, + ): + if conf.name is None: + console.print("[error]Configuration specifies no endpoint to delete[/]") + exit(1) + + try: + self.api.client.endpoints.get( + project_name=self.api.project, + name=conf.name, + ) + except ResourceNotExistsError: + console.print(f"Endpoint [code]{conf.name}[/] does not exist") + exit(1) + + if not command_args.yes and not confirm_ask(f"Delete the endpoint [code]{conf.name}[/]?"): + console.print("\nExiting...") + return + + with console.status("Deleting endpoint..."): + self.api.client.endpoints.delete(project_name=self.api.project, names=[conf.name]) + + console.print(f"Endpoint [code]{conf.name}[/] deleted") + + @classmethod + def register_args(cls, parser: argparse.ArgumentParser): + configuration_group = parser.add_argument_group(f"{cls.TYPE.value} Options") + configuration_group.add_argument( + "-n", + "--name", + dest="name", + help="The endpoint name", + ) + cls.register_env_args(configuration_group) + register_profile_args(parser) + + def apply_args(self, conf: EndpointConfiguration, args: argparse.Namespace): + profile = load_profile(Path.cwd(), args.profile) + _apply_profile(conf, profile) + apply_profile_args(args, conf) + if args.name: + conf.name = args.name + self.apply_env_vars(conf.env, args) + + def _follow_endpoint_apply(self, endpoint: Endpoint): + try: + with MultiItemStatus(_get_apply_status(endpoint), console=console) as live: + while not _is_endpoint_apply_finished(endpoint): + live.update( + get_endpoints_table([endpoint], format_date=local_time), + status=_get_apply_status(endpoint), + ) + time.sleep(2) + endpoint = self.api.client.endpoints.get( + project_name=self.api.project, + name=endpoint.name, + ) + except KeyboardInterrupt: + console.print("\nDetached") + return + + _make_endpoint_url_absolute(endpoint, self.api.client.base_url) + console.print( + get_endpoints_table( + [endpoint], + verbose=endpoint.status == EndpointStatus.RUNNING, + format_date=local_time, + ) + ) + console.print( + f"\nEndpoint [code]{endpoint.name}[/] provisioning completed " + f"[secondary]({endpoint.status.value})[/]" + ) + _print_finished_endpoint_message(endpoint) + if endpoint.status == EndpointStatus.FAILED: + exit(1) + + +def _apply_profile(conf: EndpointConfiguration, profile: Profile): + for field in ProfileParams.__fields__: + value = getattr(profile, field) + if value is not None: + setattr(conf, field, value) + + +def _is_endpoint_apply_finished(endpoint: Endpoint) -> bool: + return endpoint.status in (EndpointStatus.RUNNING, EndpointStatus.FAILED) + + +def _get_apply_status(endpoint: Endpoint) -> str: + return f"Provisioning endpoint [code]{endpoint.name}[/]..." + + +def _print_finished_endpoint_message(endpoint: Endpoint) -> None: + if endpoint.status == EndpointStatus.RUNNING: + if endpoint.url is not None: + console.print(f"[code]{endpoint.url}[/code]") + return + + message = endpoint.status_message or endpoint.error + if message is None: + message = endpoint.status.value.capitalize() + console.print(f"[error]{message}[/error]") + + +def _print_endpoint_plan(plan: EndpointPlan, no_fleets: bool = False): + def th(s: str) -> str: + return f"[bold]{s}[/bold]" + + props = Table(box=None, show_header=False) + props.add_column(no_wrap=True) + props.add_column() + props.add_row(th("Project"), plan.project_name) + props.add_row(th("User"), plan.user) + if plan.configuration_path is not None: + props.add_row(th("Configuration"), plan.configuration_path) + props.add_row(th("Type"), plan.configuration.type) + props.add_row(th("Endpoint"), plan.configuration.name or "(generated)") + props.add_row(th("Resources"), _format_resources(plan)) + props.add_row(th("Spot policy"), _format_spot_policy(plan)) + props.add_row(th("Max price"), _format_max_price(plan)) + props.add_row(th("Preset policy"), plan.preset_policy.value) + if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): + props.add_row(th("Preset"), plan.provisioning_plan.preset_name) + elif ( + isinstance(plan.provisioning_plan, EndpointProvisioningPlanAgent) + and plan.provisioning_plan.max_budget is not None + ): + props.add_row(th("Agent budget"), _format_price_limit(plan.provisioning_plan.max_budget)) + console.print(props) + console.print() + + if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): + _print_preset_plan_offers(plan.provisioning_plan, no_fleets=no_fleets) + elif isinstance(plan.provisioning_plan, EndpointProvisioningPlanNone): + style = "error" if plan.preset_policy == EndpointPresetPolicy.CREATE else "warning" + console.print(f"[{style}]{plan.provisioning_plan.reason}[/]") + console.print() + elif isinstance(plan.provisioning_plan, EndpointProvisioningPlanAgent): + if plan.provisioning_plan.reason is not None: + console.print(f"[warning]{plan.provisioning_plan.reason}[/]") + console.print() + + +def _get_apply_confirm_message( + plan: EndpointPlan, + stop_run_name: str | None = None, +) -> str: + current_resource = _get_non_terminal_current_resource(plan) + if current_resource is not None: + return f"Stop and override the endpoint [code]{current_resource.name}[/]?" + if stop_run_name is not None: + return f"Stop and override the run [code]{stop_run_name}[/]?" + return "Create the endpoint?" + + +def _get_endpoint_serving_run_name(plan: EndpointPlan) -> str | None: + if _get_non_terminal_current_resource(plan) is not None: + return None + if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): + return plan.provisioning_plan.service_name + return None + + +def _get_non_terminal_current_resource(plan: EndpointPlan) -> Endpoint | None: + if plan.current_resource is None or plan.current_resource.status.is_finished(): + return None + return plan.current_resource + + +def _format_resources(plan: EndpointPlan) -> str: + if not isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): + return "-" + resources = [ + (job_offers.replica_group, job_offers.resources.pretty_format()) + for job_offers in plan.provisioning_plan.job_offers + ] + if not resources: + return "-" + resource_values = {resource for _, resource in resources} + if len(resource_values) == 1: + return resources[0][1] + return "\n".join(f"{replica_group}: {resource}" for replica_group, resource in resources) + + +def _format_spot_policy(plan: EndpointPlan) -> str: + if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): + values = {job_offers.spot for job_offers in plan.provisioning_plan.job_offers} + if len(values) == 1: + return _format_spot(next(iter(values))) + return "mixed" + return _format_endpoint_spot_policy(plan.configuration.spot_policy) + + +def _format_endpoint_spot_policy(spot_policy: SpotPolicy | None) -> str: + if spot_policy == SpotPolicy.SPOT: + return "spot" + if spot_policy == SpotPolicy.ONDEMAND: + return "on-demand" + return "auto" + + +def _format_spot(spot: bool | None) -> str: + if spot is None: + return "auto" + return "spot" if spot else "on-demand" + + +def _format_max_price(plan: EndpointPlan) -> str: + if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): + values = {job_offers.max_price for job_offers in plan.provisioning_plan.job_offers} + if len(values) == 1: + return _format_price_limit(next(iter(values))) + return "mixed" + return _format_price_limit(plan.configuration.max_price) + + +def _format_price_limit(max_price: float | None) -> str: + return f"${max_price:3f}".rstrip("0").rstrip(".") if max_price else "off" + + +def _preset_plan_has_no_offers( + provisioning_plan: AnyEndpointProvisioningPlan, +) -> bool: + if not isinstance(provisioning_plan, EndpointProvisioningPlanPreset): + return False + return not any(job_offers.offers for job_offers in provisioning_plan.job_offers) + + +def _print_preset_plan_offers( + provisioning_plan: EndpointProvisioningPlanPreset, + no_fleets: bool = False, +) -> None: + if _preset_plan_has_no_offers(provisioning_plan): + console.print(_NO_ENDPOINT_FLEETS_WARNING if no_fleets else NO_OFFERS_WARNING) + return + + show_replica_groups = len(provisioning_plan.job_offers) > 1 or any( + job_offers.replica_group != "0" for job_offers in provisioning_plan.job_offers + ) + offers = Table(box=None, expand=shutil.get_terminal_size(fallback=(120, 40)).columns <= 110) + offers.add_column("#") + if show_replica_groups: + offers.add_column("GROUP", no_wrap=True) + offers.add_column("BACKEND", style="grey58") + offers.add_column("RESOURCES") + offers.add_column("INSTANCE TYPE", style="grey58", no_wrap=True) + offers.add_column("PRICE", style="grey58") + offers.add_column() + offer_num = 0 + for job_offers in provisioning_plan.job_offers: + for offer in job_offers.offers: + offer_num += 1 + row = [ + str(offer_num), + format_backend(offer.backend, offer.region), + offer.instance.resources.pretty_format(include_spot=True), + offer.instance.name, + f"${offer.price:.4f}".rstrip("0").rstrip("."), + format_instance_availability(offer.availability), + ] + if show_replica_groups: + row.insert(1, job_offers.replica_group) + offers.add_row(*row, style=None if offer_num == 1 else "secondary") + if job_offers.total_offers > len(job_offers.offers): + row = ["", "..."] + if show_replica_groups: + row.insert(1, job_offers.replica_group) + offers.add_row(*row, style="secondary") + console.print(offers) + console.print() + + +def _print_submitted_endpoint_message(endpoint: Endpoint) -> None: + console.print(f"Endpoint [code]{endpoint.name}[/] submitted, detaching...") + + +def _make_endpoint_url_absolute(endpoint: Endpoint, server_url: str) -> None: + if endpoint.url is None: + return + endpoint.url = make_proxy_url(server_url=server_url, proxy_url=endpoint.url) diff --git a/src/dstack/_internal/cli/services/profile.py b/src/dstack/_internal/cli/services/profile.py index e8086bb43b..63f2b4fbe5 100644 --- a/src/dstack/_internal/cli/services/profile.py +++ b/src/dstack/_internal/cli/services/profile.py @@ -6,6 +6,7 @@ from dstack._internal.core.models.profiles import ( CreationPolicy, Profile, + ProfileParams, ProfileRetry, SpotPolicy, parse_duration, @@ -135,7 +136,7 @@ def register_profile_args(parser: argparse.ArgumentParser): def apply_profile_args( args: argparse.Namespace, - profile_settings: Union[Profile, AnyRunConfiguration], + profile_settings: Union[Profile, ProfileParams, AnyRunConfiguration], ): """ Overrides `profile_settings` settings with arguments registered by `register_profile_args()`. diff --git a/src/dstack/_internal/cli/utils/endpoint.py b/src/dstack/_internal/cli/utils/endpoint.py new file mode 100644 index 0000000000..e5c8acaa2f --- /dev/null +++ b/src/dstack/_internal/cli/utils/endpoint.py @@ -0,0 +1,91 @@ +from typing import List + +from rich.table import Table + +from dstack._internal.cli.utils.common import add_row_from_dict, console +from dstack._internal.core.models.endpoints import Endpoint, EndpointStatus +from dstack._internal.utils.common import DateFormatter, pretty_date + + +def filter_endpoints_for_listing( + endpoints: List[Endpoint], + show_all: bool = False, + limit: int | None = None, + include_latest_finished: bool = True, +) -> List[Endpoint]: + endpoints = sorted( + endpoints, + key=lambda endpoint: (endpoint.created_at, str(endpoint.id)), + reverse=True, + ) + if limit is not None: + return endpoints[:limit] + if show_all: + return endpoints + + latest_finished = None + filtered = [] + for endpoint in endpoints: + if endpoint.status.is_finished(): + if not include_latest_finished: + continue + if latest_finished is None: + latest_finished = endpoint + filtered.append(endpoint) + continue + filtered.append(endpoint) + return filtered + + +def print_endpoints_table(endpoints: List[Endpoint], verbose: bool = False): + table = get_endpoints_table(endpoints, verbose=verbose) + console.print(table) + console.print() + + +def get_endpoints_table( + endpoints: List[Endpoint], + verbose: bool = False, + format_date: DateFormatter = pretty_date, +) -> Table: + table = Table(box=None) + table.add_column("NAME", no_wrap=True) + table.add_column("MODEL") + table.add_column("STATUS") + table.add_column("RUN") + if verbose: + table.add_column("URL") + table.add_column("CREATED") + if verbose: + table.add_column("ERROR") + + for endpoint in endpoints: + row = { + "NAME": endpoint.name, + "MODEL": endpoint.configuration.model, + "STATUS": _format_endpoint_status(endpoint.status), + "RUN": endpoint.run_name or "-", + "URL": endpoint.url or "-", + "CREATED": format_date(endpoint.created_at), + "ERROR": endpoint.status_message, + } + add_row_from_dict(table, row) + return table + + +def _format_endpoint_status(status: EndpointStatus) -> str: + color_map = { + EndpointStatus.SUBMITTED: "grey", + EndpointStatus.PROVISIONING: "deep_sky_blue1", + EndpointStatus.AGENTING: "medium_purple1", + EndpointStatus.RUNNING: "sea_green3", + EndpointStatus.ACTIVE: "sea_green3", + EndpointStatus.FAILED: "indian_red1", + } + style = color_map.get(status, "white") + if not status.is_finished(): + style = f"bold {style}" + status_value = ( + EndpointStatus.RUNNING.value if status == EndpointStatus.ACTIVE else status.value + ) + return f"[{style}]{status_value}[/]" diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 900dca8ce7..2240d5c23e 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -18,6 +18,7 @@ RegistryAuth, generate_dual_core_model, ) +from dstack._internal.core.models.endpoints import EndpointConfiguration from dstack._internal.core.models.envs import Env from dstack._internal.core.models.files import FilePathMapping from dstack._internal.core.models.fleets import FleetConfiguration @@ -1385,6 +1386,7 @@ class ApplyConfigurationType(str, Enum): DEV_ENVIRONMENT = "dev-environment" TASK = "task" SERVICE = "service" + ENDPOINT = "endpoint" FLEET = "fleet" GATEWAY = "gateway" VOLUME = "volume" @@ -1392,6 +1394,7 @@ class ApplyConfigurationType(str, Enum): AnyApplyConfiguration = Union[ AnyRunConfiguration, + EndpointConfiguration, FleetConfiguration, GatewayConfiguration, AnyVolumeConfiguration, @@ -1412,6 +1415,7 @@ class BaseApplyConfiguration(CoreModel): Union[ # Final configurations AnyRunConfiguration, + EndpointConfiguration, FleetConfiguration, GatewayConfiguration, # Base configurations (further parsing required to get a concrete AnyApplyConfiguration) @@ -1439,6 +1443,7 @@ def parse_apply_configuration(data: dict) -> AnyApplyConfiguration: AnyDstackConfiguration = Union[ AnyRunConfiguration, + EndpointConfiguration, FleetConfiguration, GatewayConfiguration, VolumeConfiguration, diff --git a/src/dstack/_internal/core/models/endpoints.py b/src/dstack/_internal/core/models/endpoints.py new file mode 100644 index 0000000000..09153ad6b8 --- /dev/null +++ b/src/dstack/_internal/core/models/endpoints.py @@ -0,0 +1,154 @@ +import uuid +from datetime import datetime +from enum import Enum +from typing import Annotated, Literal, Optional, Union + +from pydantic import Field + +from dstack._internal.core.models.common import ( + ApplyAction, + CoreModel, + generate_dual_core_model, +) +from dstack._internal.core.models.envs import Env +from dstack._internal.core.models.instances import InstanceOfferWithAvailability +from dstack._internal.core.models.profiles import ProfileParams, ProfileParamsConfig +from dstack._internal.core.models.resources import ResourcesSpec + + +class EndpointStatus(str, Enum): + SUBMITTED = "submitted" + PROVISIONING = "provisioning" + AGENTING = "agenting" + RUNNING = "running" + # Legacy status used by early endpoint prototypes. New code writes RUNNING. + ACTIVE = "active" + FAILED = "failed" + + @classmethod + def finished_statuses(cls) -> list["EndpointStatus"]: + return [cls.FAILED] + + def is_finished(self) -> bool: + return self in self.finished_statuses() + + +class EndpointPresetPolicy(str, Enum): + REUSE = "reuse" + CREATE = "create" + REUSE_OR_CREATE = "reuse-or-create" + + +class EndpointConfigurationConfig(ProfileParamsConfig): + @staticmethod + def schema_extra(schema: dict): + ProfileParamsConfig.schema_extra(schema) + + +class EndpointConfiguration( + ProfileParams, + generate_dual_core_model(EndpointConfigurationConfig), +): + type: Literal["endpoint"] = "endpoint" + name: Annotated[ + Optional[str], + Field(description="The endpoint name. If not specified, a random name is generated"), + ] = None + model: Annotated[ + str, + Field(description="The model to serve, typically a Hugging Face model ID"), + ] + env: Annotated[ + Env, + Field(description="The mapping or the list of environment variables"), + ] = Env() + preset_policy: Annotated[ + EndpointPresetPolicy, + Field( + description=( + "The policy for endpoint presets. `reuse` uses an existing preset only, " + "`create` asks the server agent to create and save a new preset, and " + "`reuse-or-create` first tries an existing preset and falls back to `create`." + ) + ), + ] = EndpointPresetPolicy.REUSE_OR_CREATE + max_agent_budget: Annotated[ + Optional[float], + Field( + description=( + "The maximum agent spend for provisioning this endpoint, in dollars. " + "If not specified, the server default is used." + ), + gt=0.0, + ), + ] = None + + +class Endpoint(CoreModel): + id: uuid.UUID + name: str + project_name: str + user: str + configuration: EndpointConfiguration + created_at: datetime + last_processed_at: datetime + status: EndpointStatus + status_message: Optional[str] = None + deleted: bool + deleted_at: Optional[datetime] = None + run_name: Optional[str] = None + url: Optional[str] = None + error: Optional[str] = None + + +class EndpointProvisioningPlanNone(CoreModel): + type: Literal["none"] = "none" + reason: str + + +class EndpointPlanReplicaSpecGroup(CoreModel): + name: str + replica_specs: list[ResourcesSpec] + + +class EndpointPlanJobOffers(CoreModel): + replica_group: str + resources: ResourcesSpec + spot: Optional[bool] + max_price: Optional[float] + offers: list[InstanceOfferWithAvailability] + total_offers: int + max_offer_price: Optional[float] + + +class EndpointProvisioningPlanPreset(CoreModel): + type: Literal["preset"] = "preset" + preset_name: str + service_name: str + replica_spec_groups: list[EndpointPlanReplicaSpecGroup] + job_offers: list[EndpointPlanJobOffers] + + +class EndpointProvisioningPlanAgent(CoreModel): + type: Literal["agent"] = "agent" + agent_model: str + max_budget: Optional[float] = None + reason: Optional[str] = None + + +AnyEndpointProvisioningPlan = Union[ + EndpointProvisioningPlanNone, + EndpointProvisioningPlanPreset, + EndpointProvisioningPlanAgent, +] + + +class EndpointPlan(CoreModel): + project_name: str + user: str + configuration: EndpointConfiguration + configuration_path: Optional[str] = None + current_resource: Optional[Endpoint] = None + action: ApplyAction + preset_policy: EndpointPresetPolicy + provisioning_plan: Annotated[AnyEndpointProvisioningPlan, Field(discriminator="type")] diff --git a/src/dstack/_internal/core/models/events.py b/src/dstack/_internal/core/models/events.py index f2efb80d0e..11f3881725 100644 --- a/src/dstack/_internal/core/models/events.py +++ b/src/dstack/_internal/core/models/events.py @@ -16,6 +16,7 @@ class EventTargetType(str, Enum): INSTANCE = "instance" RUN = "run" JOB = "job" + ENDPOINT = "endpoint" VOLUME = "volume" GATEWAY = "gateway" SECRET = "secret" diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 0f02806aa4..918757282f 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -29,6 +29,7 @@ from dstack._internal.server.routers import ( auth, backends, + endpoints, events, exports, files, @@ -255,6 +256,8 @@ def register_routes(app: FastAPI, ui: bool = True): app.include_router(logs.router) app.include_router(secrets.router) app.include_router(gateways.router) + app.include_router(endpoints.root_router) + app.include_router(endpoints.project_router) app.include_router(volumes.root_router) app.include_router(volumes.project_router) app.include_router(service_proxy.router, prefix="/proxy/services", tags=["proxy"]) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/__init__.py b/src/dstack/_internal/server/background/pipeline_tasks/__init__.py index 2e83e780c0..d06a472481 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/__init__.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/__init__.py @@ -2,6 +2,7 @@ from dstack._internal.server.background.pipeline_tasks.base import Pipeline from dstack._internal.server.background.pipeline_tasks.compute_groups import ComputeGroupPipeline +from dstack._internal.server.background.pipeline_tasks.endpoints import EndpointPipeline from dstack._internal.server.background.pipeline_tasks.fleets import FleetPipeline from dstack._internal.server.background.pipeline_tasks.gateway_replicas import ( GatewayReplicaPipeline, @@ -44,6 +45,7 @@ def __init__(self) -> None: PlacementGroupPipeline(pipeline_hinter=self._hinter), RunPipeline(pipeline_hinter=self._hinter), ServiceRouterWorkerSyncPipeline(pipeline_hinter=self._hinter), + EndpointPipeline(pipeline_hinter=self._hinter), VolumePipeline(pipeline_hinter=self._hinter), ]: self.register_pipeline(builtin_pipeline) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py new file mode 100644 index 0000000000..cac81aa8a1 --- /dev/null +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -0,0 +1,901 @@ +import asyncio +import uuid +from dataclasses import dataclass, field +from datetime import timedelta +from typing import Optional, Sequence + +from pydantic import ValidationError +from sqlalchemy import or_, select, update +from sqlalchemy.orm import joinedload, load_only + +from dstack._internal.core.errors import ServerClientError +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.endpoints import EndpointPresetPolicy, EndpointStatus +from dstack._internal.core.models.runs import ( + ApplyRunPlanInput, + JobStatus, + RunSpec, + RunStatus, + ServiceSpec, +) +from dstack._internal.server.background.pipeline_tasks.base import ( + NOW_PLACEHOLDER, + Fetcher, + Heartbeater, + ItemUpdateMap, + Pipeline, + PipelineItem, + UpdateMapDateTime, + Worker, + log_lock_token_changed_after_processing, + log_lock_token_mismatch, + resolve_now_placeholders, + set_processed_update_map_fields, + set_unlock_update_map_fields, +) +from dstack._internal.server.db import get_db, get_session_ctx +from dstack._internal.server.models import EndpointModel, ProjectModel, RunModel, UserModel +from dstack._internal.server.services import events +from dstack._internal.server.services import runs as runs_services +from dstack._internal.server.services.endpoints import ( + emit_endpoint_status_change_event, + get_endpoint_configuration, + record_endpoint_run_submission, +) +from dstack._internal.server.services.endpoints.agent import ( + get_agent_service, + get_agent_unavailable_reason, +) +from dstack._internal.server.services.endpoints.names import get_endpoint_serving_run_name +from dstack._internal.server.services.endpoints.planning import find_preset_planning_result +from dstack._internal.server.services.endpoints.preset_building import ( + build_endpoint_preset_from_run, +) +from dstack._internal.server.services.endpoints.presets import get_endpoint_preset_service +from dstack._internal.server.services.locking import get_locker +from dstack._internal.server.services.pipelines import PipelineHinterProtocol +from dstack._internal.server.utils import sentry_utils +from dstack._internal.utils.common import get_current_datetime +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +_NO_MATCHING_PRESET_MESSAGE = "No matching endpoint presets found." + + +@dataclass +class EndpointPipelineItem(PipelineItem): + status: EndpointStatus + to_be_deleted: bool + + +class EndpointPipeline(Pipeline[EndpointPipelineItem]): + def __init__( + self, + workers_num: int = 4, + queue_lower_limit_factor: float = 0.5, + queue_upper_limit_factor: float = 2.0, + min_processing_interval: timedelta = timedelta(seconds=10), + lock_timeout: timedelta = timedelta(seconds=30), + heartbeat_trigger: timedelta = timedelta(seconds=15), + *, + pipeline_hinter: PipelineHinterProtocol, + ) -> None: + super().__init__( + workers_num=workers_num, + queue_lower_limit_factor=queue_lower_limit_factor, + queue_upper_limit_factor=queue_upper_limit_factor, + min_processing_interval=min_processing_interval, + lock_timeout=lock_timeout, + heartbeat_trigger=heartbeat_trigger, + ) + self.__heartbeater = Heartbeater[EndpointPipelineItem]( + model_type=EndpointModel, + lock_timeout=self._lock_timeout, + heartbeat_trigger=self._heartbeat_trigger, + ) + self.__fetcher = EndpointFetcher( + queue=self._queue, + queue_desired_minsize=self._queue_desired_minsize, + min_processing_interval=self._min_processing_interval, + lock_timeout=self._lock_timeout, + heartbeater=self._heartbeater, + ) + self.__workers = [ + EndpointWorker( + queue=self._queue, + heartbeater=self._heartbeater, + pipeline_hinter=pipeline_hinter, + ) + for _ in range(self._workers_num) + ] + + @property + def hint_fetch_model_name(self) -> str: + return EndpointModel.__name__ + + @property + def _heartbeater(self) -> Heartbeater[EndpointPipelineItem]: + return self.__heartbeater + + @property + def _fetcher(self) -> Fetcher[EndpointPipelineItem]: + return self.__fetcher + + @property + def _workers(self) -> Sequence["EndpointWorker"]: + return self.__workers + + +class EndpointFetcher(Fetcher[EndpointPipelineItem]): + def __init__( + self, + queue: asyncio.Queue[EndpointPipelineItem], + queue_desired_minsize: int, + min_processing_interval: timedelta, + lock_timeout: timedelta, + heartbeater: Heartbeater[EndpointPipelineItem], + queue_check_delay: float = 1.0, + ) -> None: + super().__init__( + queue=queue, + queue_desired_minsize=queue_desired_minsize, + min_processing_interval=min_processing_interval, + lock_timeout=lock_timeout, + heartbeater=heartbeater, + queue_check_delay=queue_check_delay, + ) + + @sentry_utils.instrument_pipeline_task("EndpointFetcher.fetch") + async def fetch(self, limit: int) -> list[EndpointPipelineItem]: + endpoint_lock, _ = get_locker(get_db().dialect_name).get_lockset( + EndpointModel.__tablename__ + ) + async with endpoint_lock: + async with get_session_ctx() as session: + now = get_current_datetime() + res = await session.execute( + select(EndpointModel) + .where( + or_( + EndpointModel.status.in_( + [ + EndpointStatus.SUBMITTED, + EndpointStatus.PROVISIONING, + EndpointStatus.AGENTING, + EndpointStatus.RUNNING, + EndpointStatus.ACTIVE, + ] + ), + EndpointModel.to_be_deleted == True, + ), + EndpointModel.deleted == False, + or_( + EndpointModel.last_processed_at <= now - self._min_processing_interval, + EndpointModel.last_processed_at == EndpointModel.created_at, + ), + or_( + EndpointModel.lock_expires_at.is_(None), + EndpointModel.lock_expires_at < now, + ), + or_( + EndpointModel.lock_owner.is_(None), + EndpointModel.lock_owner == EndpointPipeline.__name__, + ), + ) + .order_by(EndpointModel.last_processed_at.asc()) + .limit(limit) + .with_for_update(skip_locked=True, key_share=True, of=EndpointModel) + .options( + load_only( + EndpointModel.id, + EndpointModel.lock_token, + EndpointModel.lock_expires_at, + EndpointModel.status, + EndpointModel.to_be_deleted, + ) + ) + ) + endpoint_models = list(res.scalars().all()) + lock_expires_at = get_current_datetime() + self._lock_timeout + lock_token = uuid.uuid4() + items = [] + for endpoint_model in endpoint_models: + prev_lock_expired = endpoint_model.lock_expires_at is not None + endpoint_model.lock_expires_at = lock_expires_at + endpoint_model.lock_token = lock_token + endpoint_model.lock_owner = EndpointPipeline.__name__ + items.append( + EndpointPipelineItem( + __tablename__=EndpointModel.__tablename__, + id=endpoint_model.id, + lock_expires_at=lock_expires_at, + lock_token=lock_token, + prev_lock_expired=prev_lock_expired, + status=endpoint_model.status, + to_be_deleted=endpoint_model.to_be_deleted, + ) + ) + await session.commit() + return items + + +class EndpointWorker(Worker[EndpointPipelineItem]): + def __init__( + self, + queue: asyncio.Queue[EndpointPipelineItem], + heartbeater: Heartbeater[EndpointPipelineItem], + pipeline_hinter: PipelineHinterProtocol, + ) -> None: + super().__init__( + queue=queue, + heartbeater=heartbeater, + pipeline_hinter=pipeline_hinter, + ) + + @sentry_utils.instrument_pipeline_task("EndpointWorker.process") + async def process(self, item: EndpointPipelineItem): + endpoint_model = await _refetch_locked_endpoint(item) + if endpoint_model is None: + log_lock_token_mismatch(logger, item) + return + + if endpoint_model.to_be_deleted: + result = await _process_to_be_deleted_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=self._pipeline_hinter, + ) + elif endpoint_model.status == EndpointStatus.SUBMITTED: + result = await _process_submitted_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=self._pipeline_hinter, + ) + elif endpoint_model.status == EndpointStatus.PROVISIONING: + result = await _process_provisioning_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=self._pipeline_hinter, + ) + elif endpoint_model.status == EndpointStatus.AGENTING: + result = await _process_agenting_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=self._pipeline_hinter, + ) + elif endpoint_model.status in (EndpointStatus.RUNNING, EndpointStatus.ACTIVE): + result = await _process_running_endpoint(endpoint_model) + else: + result = _ProcessResult() + + run_to_stop = await _get_backing_run_to_stop_after_failure(endpoint_model, result) + if run_to_stop is not None: + logger.info( + "Stopping backing run %s after endpoint %s failed", + run_to_stop.run_name, + endpoint_model.name, + ) + await _stop_backing_run( + endpoint_model=endpoint_model, + run_name=run_to_stop.run_name, + pipeline_hinter=self._pipeline_hinter, + ) + + await _apply_process_result(item=item, endpoint_model=endpoint_model, result=result) + + +async def _refetch_locked_endpoint(item: EndpointPipelineItem) -> Optional[EndpointModel]: + async with get_session_ctx() as session: + res = await session.execute( + select(EndpointModel) + .where( + EndpointModel.id == item.id, + EndpointModel.lock_token == item.lock_token, + ) + .options(joinedload(EndpointModel.project).joinedload(ProjectModel.backends)) + .options(joinedload(EndpointModel.user).load_only(UserModel.name, UserModel.token)) + .options(joinedload(EndpointModel.service_run).selectinload(RunModel.jobs)) + ) + return res.unique().scalar_one_or_none() + + +async def _apply_process_result( + item: EndpointPipelineItem, + endpoint_model: EndpointModel, + result: "_ProcessResult", +): + update_map = _EndpointUpdateMap() + update_map.update(result.update_map) + set_processed_update_map_fields(update_map) + set_unlock_update_map_fields(update_map) + + async with get_session_ctx() as session: + resolve_now_placeholders(update_map, now=get_current_datetime()) + res = await session.execute( + update(EndpointModel) + .where( + EndpointModel.id == endpoint_model.id, + EndpointModel.lock_token == endpoint_model.lock_token, + ) + .values(**update_map) + .returning(EndpointModel.id) + ) + updated_ids = list(res.scalars().all()) + if len(updated_ids) == 0: + log_lock_token_changed_after_processing(logger, item) + return + if result.update_map.get("deleted"): + events.emit( + session, + "Endpoint deleted", + actor=events.SystemActor(), + targets=[events.Target.from_model(endpoint_model)], + ) + else: + emit_endpoint_status_change_event( + session=session, + endpoint_model=endpoint_model, + old_status=endpoint_model.status, + new_status=update_map.get("status", endpoint_model.status), + status_message=update_map.get("status_message", endpoint_model.status_message), + ) + + +class _EndpointUpdateMap(ItemUpdateMap, total=False): + status: EndpointStatus + status_message: Optional[str] + service_run_id: uuid.UUID + provisioning_method: Optional[str] + deleted: bool + deleted_at: UpdateMapDateTime + + +@dataclass +class _ProcessResult: + update_map: _EndpointUpdateMap = field(default_factory=_EndpointUpdateMap) + + +@dataclass(frozen=True) +class _PresetSubmission: + run_id: uuid.UUID + preset_name: str + + +@dataclass(frozen=True) +class _PresetSubmissionResult: + submission: Optional[_PresetSubmission] = None + unprovisionable_preset_name: Optional[str] = None + + +async def _get_backing_run_to_stop_after_failure( + endpoint_model: EndpointModel, + result: _ProcessResult, +) -> Optional[RunModel]: + if result.update_map.get("status") != EndpointStatus.FAILED: + return None + run_model = endpoint_model.service_run + if run_model is None or run_model.deleted: + return None + if run_model.status.is_finished() or run_model.status == RunStatus.TERMINATING: + return None + return run_model + + +async def _process_submitted_endpoint( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: + conflict_result = await _get_active_serving_run_name_conflict(endpoint_model) + if conflict_result is not None: + return conflict_result + + try: + submission_result = await _submit_endpoint_from_preset( + endpoint_id=endpoint_model.id, + pipeline_hinter=pipeline_hinter, + ) + except ServerClientError as e: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": e.msg, + } + ) + if submission_result.submission is not None: + logger.info( + "Provisioning endpoint %s from preset %s", + endpoint_model.name, + submission_result.submission.preset_name, + ) + update_map = _EndpointUpdateMap( + status=EndpointStatus.PROVISIONING, + status_message=None, + service_run_id=submission_result.submission.run_id, + provisioning_method=f"preset:{submission_result.submission.preset_name}", + ) + return _ProcessResult(update_map=update_map) + + if _should_provision_with_agent(endpoint_model): + logger.info("Provisioning endpoint %s with server agent", endpoint_model.name) + update_map = _EndpointUpdateMap( + status=EndpointStatus.AGENTING, + status_message=None, + provisioning_method="agent", + ) + return _ProcessResult(update_map=update_map) + + logger.info("Failing endpoint %s: no preset path is available", endpoint_model.name) + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": _get_no_provisioning_path_message( + endpoint_model, + unprovisionable_preset_name=submission_result.unprovisionable_preset_name, + ), + } + ) + + +def _should_provision_with_agent(endpoint_model: EndpointModel) -> bool: + endpoint_configuration = get_endpoint_configuration(endpoint_model) + return ( + endpoint_configuration.preset_policy != EndpointPresetPolicy.REUSE + and get_agent_service().is_enabled() + ) + + +async def _get_active_serving_run_name_conflict( + endpoint_model: EndpointModel, +) -> Optional[_ProcessResult]: + serving_run_name = get_endpoint_serving_run_name(endpoint_model.name) + assert serving_run_name is not None + async with get_session_ctx() as session: + run_model = await runs_services.get_run_model_by_name( + session=session, + project=endpoint_model.project, + run_name=serving_run_name, + ) + if run_model is None: + return None + if endpoint_model.service_run_id == run_model.id: + return None + if run_model.status.is_finished(): + return None + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": f"Run name '{serving_run_name}' is taken by an existing run", + } + ) + + +async def _process_provisioning_endpoint( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: + if endpoint_model.service_run is None: + if endpoint_model.provisioning_method == "agent": + return await _process_agenting_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=pipeline_hinter, + ) + conflict_result = await _get_active_serving_run_name_conflict(endpoint_model) + if conflict_result is not None: + return conflict_result + + readiness = _get_backing_service_readiness(endpoint_model) + if readiness.failed_message is not None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": readiness.failed_message, + } + ) + if readiness.model_base_url is None or readiness.model_name is None: + return _ProcessResult() + if endpoint_model.provisioning_method == "agent": + # The agent's verification report is the functional signal. This server-side gate only + # confirms that the verified run still looks like a normal ready dstack service. + await _try_save_agent_endpoint_preset( + endpoint_model=endpoint_model, + model_name=readiness.model_name, + ) + return _ProcessResult( + update_map={ + "status": EndpointStatus.RUNNING, + "status_message": None, + } + ) + + +async def _process_agenting_endpoint( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: + if endpoint_model.service_run is not None: + return _ProcessResult(update_map={"status": EndpointStatus.PROVISIONING}) + return await _provision_endpoint_with_agent( + endpoint_model=endpoint_model, + pipeline_hinter=pipeline_hinter, + ) + + +async def _provision_endpoint_with_agent( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: + agent_service = get_agent_service() + if not agent_service.is_enabled(): + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": _get_no_provisioning_path_message(endpoint_model), + } + ) + + result = await agent_service.provision_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=pipeline_hinter, + ) + if result.error is not None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": result.error, + } + ) + report = result.final_report + if report is None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": "Server agent did not return a verification report", + } + ) + if not report.success: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": ( + report.failure_summary or "Server agent did not verify the endpoint" + ), + } + ) + run_id = result.run_id or report.run_id + if result.run_id is not None and report.run_id is not None and result.run_id != report.run_id: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": "Server agent returned inconsistent run id in verification report", + } + ) + if ( + result.run_name is not None + and report.run_name is not None + and result.run_name != report.run_name + ): + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": ( + "Server agent returned inconsistent run name in verification report" + ), + } + ) + if run_id is None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": "Server agent verification report did not identify a run id", + } + ) + + async with get_session_ctx() as session: + res = await session.execute( + select(RunModel) + .where( + RunModel.id == run_id, + RunModel.project_id == endpoint_model.project_id, + RunModel.deleted == False, + ) + .options(joinedload(RunModel.user)) + .options(joinedload(RunModel.jobs)) + ) + run_model = res.unique().scalar_one_or_none() + if run_model is None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": (f"Server agent reported run '{run_id}' but it was not found"), + } + ) + if run_model.user_id != endpoint_model.user_id: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": f"Run '{run_model.run_name}' is not owned by the endpoint user", + } + ) + try: + run_spec = RunSpec.__response__.parse_raw(run_model.run_spec) + except ValidationError: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": f"Run '{run_model.run_name}' has invalid run spec", + } + ) + if not isinstance(run_spec.configuration, ServiceConfiguration): + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": f"Run '{run_model.run_name}' is not a service", + } + ) + try: + await _record_endpoint_run_submission( + endpoint_id=endpoint_model.id, + run_id=run_model.id, + ) + except ServerClientError as e: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": e.msg, + } + ) + return _ProcessResult( + update_map={ + "status": EndpointStatus.PROVISIONING, + "status_message": None, + "service_run_id": run_model.id, + } + ) + + +async def _record_endpoint_run_submission(endpoint_id: uuid.UUID, run_id: uuid.UUID) -> None: + async with get_session_ctx() as session: + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_id, + run_id=run_id, + ) + await session.commit() + + +async def _process_running_endpoint(endpoint_model: EndpointModel) -> _ProcessResult: + readiness = _get_backing_service_readiness(endpoint_model) + if readiness.failed_message is not None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": readiness.failed_message, + } + ) + if readiness.model_base_url is None or readiness.model_name is None: + return _ProcessResult() + return _ProcessResult(update_map={"status_message": None}) + + +async def _try_save_agent_endpoint_preset( + endpoint_model: EndpointModel, + model_name: str, +) -> None: + run_model = endpoint_model.service_run + if run_model is None: + return + preset_name = f"{model_name}-{str(run_model.id)[:8]}" + try: + preset = build_endpoint_preset_from_run(name=preset_name, run_model=run_model) + saved_preset = await get_endpoint_preset_service().save_preset( + preset, + comments=[ + "Generated by dstack endpoint agent.", + f"endpoint: {endpoint_model.name}", + f"endpoint_id: {endpoint_model.id}", + f"run_id: {run_model.id}", + ], + ) + except Exception as e: + logger.warning( + "Failed to save endpoint preset for endpoint %s: %s", + endpoint_model.name, + e, + exc_info=True, + ) + return + logger.info("Saved endpoint preset %s for endpoint %s", saved_preset.name, endpoint_model.name) + + +@dataclass(frozen=True) +class _BackingServiceReadiness: + failed_message: Optional[str] = None + model_base_url: Optional[str] = None + model_name: Optional[str] = None + + +def _get_backing_service_readiness(endpoint_model: EndpointModel) -> _BackingServiceReadiness: + run_model = endpoint_model.service_run + if run_model is None: + return _BackingServiceReadiness(failed_message="Backing service run is missing") + if run_model.deleted: + return _BackingServiceReadiness(failed_message="Backing service run was deleted") + if run_model.status.is_finished(): + return _BackingServiceReadiness( + failed_message=f"Backing service run finished with status {run_model.status.value}" + ) + if run_model.status != RunStatus.RUNNING: + return _BackingServiceReadiness() + if not _has_registered_running_job(run_model): + return _BackingServiceReadiness() + if run_model.service_spec is None: + return _BackingServiceReadiness() + try: + service_spec = ServiceSpec.__response__.parse_raw(run_model.service_spec) + except ValidationError: + logger.warning("Endpoint %s backing service spec is invalid", endpoint_model.name) + return _BackingServiceReadiness(failed_message="Backing service spec is invalid") + if service_spec.model is None: + return _BackingServiceReadiness() + return _BackingServiceReadiness( + model_base_url=service_spec.model.base_url, + model_name=service_spec.model.name, + ) + + +def _has_registered_running_job(run_model: RunModel) -> bool: + return any(job.status == JobStatus.RUNNING and job.registered for job in run_model.jobs) + + +async def _submit_endpoint_from_preset( + endpoint_id: uuid.UUID, + pipeline_hinter: PipelineHinterProtocol, +) -> _PresetSubmissionResult: + async with get_session_ctx() as session: + res = await session.execute( + select(EndpointModel) + .where( + EndpointModel.id == endpoint_id, + EndpointModel.deleted == False, + EndpointModel.to_be_deleted == False, + EndpointModel.status == EndpointStatus.SUBMITTED, + ) + .options(joinedload(EndpointModel.project).joinedload(ProjectModel.backends)) + .options(joinedload(EndpointModel.user)) + ) + endpoint_model = res.unique().scalar_one_or_none() + if endpoint_model is None: + return _PresetSubmissionResult() + endpoint_configuration = get_endpoint_configuration(endpoint_model) + if endpoint_configuration.preset_policy == EndpointPresetPolicy.CREATE: + return _PresetSubmissionResult() + preset_planning_result = await find_preset_planning_result( + session=session, + project=endpoint_model.project, + user=endpoint_model.user, + endpoint_name=endpoint_model.name, + endpoint_configuration=endpoint_configuration, + ) + preset_plan = preset_planning_result.provisionable + if preset_plan is None: + unprovisionable_preset_name = None + if preset_planning_result.unprovisionable is not None: + unprovisionable_preset_name = preset_planning_result.unprovisionable.preset.name + return _PresetSubmissionResult(unprovisionable_preset_name=unprovisionable_preset_name) + run = await runs_services.apply_plan( + session=session, + user=endpoint_model.user, + project=endpoint_model.project, + plan=ApplyRunPlanInput( + run_spec=preset_plan.run_plan.run_spec, + current_resource=preset_plan.run_plan.current_resource, + ), + force=False, + pipeline_hinter=pipeline_hinter, + ) + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run.id, + ) + await session.commit() + return _PresetSubmissionResult( + submission=_PresetSubmission(run_id=run.id, preset_name=preset_plan.preset.name) + ) + + +def _get_no_provisioning_path_message( + endpoint_model: EndpointModel, + unprovisionable_preset_name: Optional[str] = None, +) -> str: + endpoint_configuration = get_endpoint_configuration(endpoint_model) + if unprovisionable_preset_name is not None: + reason = ( + f"Endpoint preset {unprovisionable_preset_name} matched but has no available offers." + ) + if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE: + return reason + agent_unavailable_reason = get_agent_unavailable_reason() + if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: + return ( + f"{reason} Creating a preset requires the server agent, " + f"but {agent_unavailable_reason}" + ) + if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE: + return _NO_MATCHING_PRESET_MESSAGE + agent_unavailable_reason = get_agent_unavailable_reason() + if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: + return ( + "No matching endpoint presets found. " + f"Creating a preset requires the server agent, but {agent_unavailable_reason}" + ) + return f"Preset policy create requires the server agent, but {agent_unavailable_reason}" + + +async def _process_to_be_deleted_endpoint( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: + run_model = await _get_backing_run_for_deletion(endpoint_model) + if run_model is None: + return _get_deleted_result() + if not run_model.status.is_finished(): + if run_model.status != RunStatus.TERMINATING: + logger.info( + "Stopping backing run %s before deleting endpoint %s", + run_model.run_name, + endpoint_model.name, + ) + await _stop_backing_run( + endpoint_model=endpoint_model, + run_name=run_model.run_name, + pipeline_hinter=pipeline_hinter, + ) + return _ProcessResult() + + logger.info( + "Deleting finished backing run %s before deleting endpoint %s", + run_model.run_name, + endpoint_model.name, + ) + await _delete_backing_run(endpoint_model=endpoint_model, run_name=run_model.run_name) + return _get_deleted_result() + + +async def _get_backing_run_for_deletion(endpoint_model: EndpointModel) -> Optional[RunModel]: + run_model = endpoint_model.service_run + if run_model is not None and not run_model.deleted: + return run_model + return None + + +async def _stop_backing_run( + endpoint_model: EndpointModel, + run_name: str, + pipeline_hinter: PipelineHinterProtocol, +) -> None: + async with get_session_ctx() as session: + await runs_services.stop_runs( + session=session, + user=endpoint_model.user, + project=endpoint_model.project, + runs_names=[run_name], + abort=False, + pipeline_hinter=pipeline_hinter, + ) + + +async def _delete_backing_run(endpoint_model: EndpointModel, run_name: str) -> None: + async with get_session_ctx() as session: + await runs_services.delete_runs( + session=session, + user=endpoint_model.user, + project=endpoint_model.project, + runs_names=[run_name], + ) + + +def _get_deleted_result() -> _ProcessResult: + return _ProcessResult( + update_map={ + "deleted": True, + "deleted_at": NOW_PLACEHOLDER, + } + ) diff --git a/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py b/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py new file mode 100644 index 0000000000..f6f8798aff --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py @@ -0,0 +1,127 @@ +"""add endpoints + +Revision ID: 03efb71a1563 +Revises: e9c5e7e26c78 +Create Date: 2026-07-03 12:43:58.140371+00:00 + +""" + +import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op + +import dstack._internal.server.models + +# revision identifiers, used by Alembic. +revision = "03efb71a1563" +down_revision = "e9c5e7e26c78" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "endpoints", + sa.Column("id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False), + sa.Column("name", sa.String(length=100), nullable=False), + sa.Column( + "project_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False + ), + sa.Column("user_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False), + sa.Column( + "service_run_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=True + ), + sa.Column("configuration", sa.Text(), nullable=False), + sa.Column("status", sa.String(length=100), nullable=False), + sa.Column("status_message", sa.Text(), nullable=True), + sa.Column("provisioning_method", sa.String(length=100), nullable=True), + sa.Column("created_at", dstack._internal.server.models.NaiveDateTime(), nullable=False), + sa.Column( + "last_processed_at", dstack._internal.server.models.NaiveDateTime(), nullable=False + ), + sa.Column("to_be_deleted", sa.Boolean(), server_default=sa.false(), nullable=False), + sa.Column( + "deletion_requested_at", dstack._internal.server.models.NaiveDateTime(), nullable=True + ), + sa.Column("deleted", sa.Boolean(), nullable=False), + sa.Column("deleted_at", dstack._internal.server.models.NaiveDateTime(), nullable=True), + sa.Column( + "lock_expires_at", dstack._internal.server.models.NaiveDateTime(), nullable=True + ), + sa.Column("lock_token", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=True), + sa.Column("lock_owner", sa.String(length=100), nullable=True), + sa.ForeignKeyConstraint( + ["project_id"], + ["projects.id"], + name=op.f("fk_endpoints_project_id_projects"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["service_run_id"], ["runs.id"], name=op.f("fk_endpoints_service_run_id_runs") + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name=op.f("fk_endpoints_user_id_users"), ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_endpoints")), + ) + with op.batch_alter_table("endpoints", schema=None) as batch_op: + batch_op.create_index( + "ix_endpoints_pipeline_fetch_q", + [sa.literal_column("last_processed_at ASC")], + unique=False, + postgresql_where=sa.text("deleted IS FALSE"), + sqlite_where=sa.text("deleted = 0"), + ) + batch_op.create_index(batch_op.f("ix_endpoints_status"), ["status"], unique=False) + + op.create_table( + "endpoint_run_submissions", + sa.Column( + "endpoint_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False + ), + sa.Column("run_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False), + sa.Column("submission_num", sa.Integer(), nullable=False), + sa.Column("submitted_at", dstack._internal.server.models.NaiveDateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["endpoint_id"], + ["endpoints.id"], + name=op.f("fk_endpoint_run_submissions_endpoint_id_endpoints"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["run_id"], + ["runs.id"], + name=op.f("fk_endpoint_run_submissions_run_id_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "endpoint_id", "submission_num", name=op.f("pk_endpoint_run_submissions") + ), + sa.UniqueConstraint("run_id", name="uq_endpoint_run_submissions_run_id"), + ) + with op.batch_alter_table("endpoint_run_submissions", schema=None) as batch_op: + batch_op.create_index( + batch_op.f("ix_endpoint_run_submissions_endpoint_id"), ["endpoint_id"], unique=False + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("endpoint_run_submissions", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_endpoint_run_submissions_endpoint_id")) + + op.drop_table("endpoint_run_submissions") + + with op.batch_alter_table("endpoints", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_endpoints_status")) + batch_op.drop_index( + "ix_endpoints_pipeline_fetch_q", + postgresql_where=sa.text("deleted IS FALSE"), + sqlite_where=sa.text("deleted = 0"), + ) + + op.drop_table("endpoints") + # ### end Alembic commands ### diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index fc73010263..979578b424 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -26,6 +26,7 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.common import CoreConfig, generate_dual_core_model from dstack._internal.core.models.compute_groups import ComputeGroupStatus +from dstack._internal.core.models.endpoints import EndpointStatus from dstack._internal.core.models.events import EventTargetType from dstack._internal.core.models.fleets import FleetStatus from dstack._internal.core.models.gateways import GatewayReplicaStatus, GatewayStatus @@ -998,6 +999,73 @@ class VolumeModel(PipelineModelMixin, BaseModel): ) +class EndpointModel(PipelineModelMixin, BaseModel): + __tablename__ = "endpoints" + + id: Mapped[uuid.UUID] = mapped_column( + UUIDType(binary=False), primary_key=True, default=uuid.uuid4 + ) + name: Mapped[str] = mapped_column(String(100)) + + project_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) + project: Mapped["ProjectModel"] = relationship(foreign_keys=[project_id]) + + user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + user: Mapped["UserModel"] = relationship() + + service_run_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("runs.id")) + service_run: Mapped[Optional["RunModel"]] = relationship() + """The service run currently backing this endpoint. + + This is the current/latest run. Historical endpoint-submitted runs are + recorded in EndpointRunSubmissionModel. + """ + + configuration: Mapped[str] = mapped_column(Text) + status: Mapped[EndpointStatus] = mapped_column(EnumAsString(EndpointStatus, 100), index=True) + """`status` must be changed only via `switch_endpoint_status()`.""" + status_message: Mapped[Optional[str]] = mapped_column(Text) + provisioning_method: Mapped[Optional[str]] = mapped_column(String(100)) + + created_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime) + last_processed_at: Mapped[datetime] = mapped_column( + NaiveDateTime, default=get_current_datetime + ) + to_be_deleted: Mapped[bool] = mapped_column(Boolean, server_default=false()) + deletion_requested_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + deleted: Mapped[bool] = mapped_column(Boolean, default=False) + deleted_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + + __table_args__ = ( + Index( + "ix_endpoints_pipeline_fetch_q", + last_processed_at.asc(), + postgresql_where=deleted == false(), + sqlite_where=deleted == false(), + ), + ) + + +class EndpointRunSubmissionModel(BaseModel): + __tablename__ = "endpoint_run_submissions" + + endpoint_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("endpoints.id", ondelete="CASCADE"), primary_key=True + ) + endpoint: Mapped["EndpointModel"] = relationship() + + run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) + run: Mapped["RunModel"] = relationship() + + submission_num: Mapped[int] = mapped_column(Integer, primary_key=True) + submitted_at: Mapped[datetime] = mapped_column(NaiveDateTime) + + __table_args__ = ( + UniqueConstraint("run_id", name="uq_endpoint_run_submissions_run_id"), + Index("ix_endpoint_run_submissions_endpoint_id", endpoint_id), + ) + + class VolumeAttachmentModel(BaseModel): __tablename__ = "volumes_attachments" diff --git a/src/dstack/_internal/server/routers/endpoints.py b/src/dstack/_internal/server/routers/endpoints.py new file mode 100644 index 0000000000..1e1209cd29 --- /dev/null +++ b/src/dstack/_internal/server/routers/endpoints.py @@ -0,0 +1,132 @@ +from typing import List, Tuple + +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +import dstack._internal.server.services.endpoints as endpoints_services +from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.core.models.endpoints import Endpoint, EndpointPlan +from dstack._internal.server.db import get_session +from dstack._internal.server.models import ProjectModel, UserModel +from dstack._internal.server.schemas.endpoints import ( + CreateEndpointRequest, + DeleteEndpointsRequest, + GetEndpointPlanRequest, + GetEndpointRequest, + ListEndpointsRequest, +) +from dstack._internal.server.security.permissions import Authenticated, ProjectMember +from dstack._internal.server.services.pipelines import PipelineHinterProtocol, get_pipeline_hinter +from dstack._internal.server.utils.routers import ( + CustomORJSONResponse, + get_base_api_additional_responses, +) + +root_router = APIRouter( + prefix="/api/endpoints", + tags=["endpoints"], + responses=get_base_api_additional_responses(), +) +project_router = APIRouter(prefix="/api/project/{project_name}/endpoints", tags=["endpoints"]) + + +@root_router.post("/list", summary="List endpoints", response_model=List[Endpoint]) +async def list_endpoints( + body: ListEndpointsRequest, + session: AsyncSession = Depends(get_session), + user: UserModel = Depends(Authenticated()), +): + return CustomORJSONResponse( + await endpoints_services.list_endpoints( + session=session, + user=user, + project_name=body.project_name, + only_active=body.only_active, + prev_created_at=body.prev_created_at, + prev_id=body.prev_id, + limit=body.limit, + ascending=body.ascending, + ) + ) + + +@project_router.post("/list", summary="List project endpoints", response_model=List[Endpoint]) +async def list_project_endpoints( + session: AsyncSession = Depends(get_session), + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), +): + _, project = user_project + return CustomORJSONResponse( + await endpoints_services.list_project_endpoints(session=session, project=project) + ) + + +@project_router.post("/get", summary="Get endpoint", response_model=Endpoint) +async def get_endpoint( + body: GetEndpointRequest, + session: AsyncSession = Depends(get_session), + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), +): + _, project = user_project + endpoint = await endpoints_services.get_endpoint_by_name( + session=session, + project=project, + name=body.name, + ) + if endpoint is None: + raise ResourceNotExistsError() + return CustomORJSONResponse(endpoint) + + +@project_router.post("/get_plan", summary="Get endpoint apply plan", response_model=EndpointPlan) +async def get_endpoint_plan( + body: GetEndpointPlanRequest, + session: AsyncSession = Depends(get_session), + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), +): + user, project = user_project + return CustomORJSONResponse( + await endpoints_services.get_endpoint_plan( + session=session, + project=project, + user=user, + configuration=body.configuration, + configuration_path=body.configuration_path, + ) + ) + + +@project_router.post("/create", summary="Create endpoint", response_model=Endpoint) +async def create_endpoint( + body: CreateEndpointRequest, + session: AsyncSession = Depends(get_session), + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), + pipeline_hinter: PipelineHinterProtocol = Depends(get_pipeline_hinter), +): + user, project = user_project + return CustomORJSONResponse( + await endpoints_services.create_endpoint( + session=session, + project=project, + user=user, + configuration=body.configuration, + pipeline_hinter=pipeline_hinter, + ) + ) + + +@project_router.post("/delete", summary="Delete endpoints") +async def delete_endpoints( + body: DeleteEndpointsRequest, + session: AsyncSession = Depends(get_session), + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), + pipeline_hinter: PipelineHinterProtocol = Depends(get_pipeline_hinter), +): + user, project = user_project + await endpoints_services.delete_endpoints( + session=session, + project=project, + names=body.names, + user=user, + pipeline_hinter=pipeline_hinter, + ) diff --git a/src/dstack/_internal/server/schemas/endpoints.py b/src/dstack/_internal/server/schemas/endpoints.py new file mode 100644 index 0000000000..d3ecbf780c --- /dev/null +++ b/src/dstack/_internal/server/schemas/endpoints.py @@ -0,0 +1,34 @@ +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +from pydantic import Field + +from dstack._internal.core.models.common import CoreModel +from dstack._internal.core.models.endpoints import EndpointConfiguration + + +class ListEndpointsRequest(CoreModel): + project_name: Optional[str] + only_active: bool = False + prev_created_at: Optional[datetime] + prev_id: Optional[UUID] + limit: int = Field(100, ge=0, le=100) + ascending: bool = False + + +class GetEndpointRequest(CoreModel): + name: str + + +class GetEndpointPlanRequest(CoreModel): + configuration: EndpointConfiguration + configuration_path: Optional[str] = None + + +class CreateEndpointRequest(CoreModel): + configuration: EndpointConfiguration + + +class DeleteEndpointsRequest(CoreModel): + names: List[str] diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py new file mode 100644 index 0000000000..9f4e05b6db --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -0,0 +1,573 @@ +import uuid +from datetime import datetime +from typing import Any, List, Optional + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload + +from dstack._internal.core.errors import ResourceExistsError, ServerClientError +from dstack._internal.core.models.common import ApplyAction +from dstack._internal.core.models.endpoints import ( + Endpoint, + EndpointConfiguration, + EndpointPlan, + EndpointPlanJobOffers, + EndpointPlanReplicaSpecGroup, + EndpointPresetPolicy, + EndpointProvisioningPlanAgent, + EndpointProvisioningPlanNone, + EndpointProvisioningPlanPreset, + EndpointStatus, +) +from dstack._internal.core.models.runs import ServiceSpec +from dstack._internal.core.services import validate_dstack_resource_name +from dstack._internal.server.db import get_db, is_db_postgres, is_db_sqlite +from dstack._internal.server.models import ( + EndpointModel, + EndpointRunSubmissionModel, + ProjectModel, + UserModel, +) +from dstack._internal.server.services import events +from dstack._internal.server.services.endpoints.agent import ( + AgentPlan, + get_agent_service, + get_agent_unavailable_reason, + get_effective_max_agent_budget, +) +from dstack._internal.server.services.endpoints.planning import ( + EndpointPresetPlan, + find_preset_planning_result, +) +from dstack._internal.server.services.locking import get_locker, string_to_lock_id +from dstack._internal.server.services.pipelines import PipelineHinterProtocol +from dstack._internal.server.services.projects import list_user_project_models +from dstack._internal.utils import common, random_names +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +def switch_endpoint_status( + session: AsyncSession, + endpoint_model: EndpointModel, + new_status: EndpointStatus, + actor: events.AnyActor = events.SystemActor(), +): + old_status = endpoint_model.status + if old_status == new_status: + return + + endpoint_model.status = new_status + emit_endpoint_status_change_event( + session=session, + endpoint_model=endpoint_model, + old_status=old_status, + new_status=new_status, + status_message=endpoint_model.status_message, + actor=actor, + ) + + +def emit_endpoint_status_change_event( + session: AsyncSession, + endpoint_model: EndpointModel, + old_status: EndpointStatus, + new_status: EndpointStatus, + status_message: Optional[str], + actor: events.AnyActor = events.SystemActor(), +) -> None: + if old_status == new_status: + return + msg = get_endpoint_status_change_message( + old_status=old_status, + new_status=new_status, + status_message=status_message, + ) + events.emit(session, msg, actor=actor, targets=[events.Target.from_model(endpoint_model)]) + + +def get_endpoint_status_change_message( + old_status: EndpointStatus, + new_status: EndpointStatus, + status_message: Optional[str], +) -> str: + msg = f"Endpoint status changed {old_status.upper()} -> {new_status.upper()}" + if status_message is not None: + msg += f" ({status_message})" + return msg + + +async def list_endpoints( + session: AsyncSession, + user: UserModel, + project_name: Optional[str], + only_active: bool, + prev_created_at: Optional[datetime], + prev_id: Optional[uuid.UUID], + limit: int, + ascending: bool, +) -> List[Endpoint]: + projects = await list_user_project_models( + session=session, + user=user, + only_names=True, + ) + if project_name is not None: + projects = [p for p in projects if p.name == project_name] + endpoint_models = await list_projects_endpoint_models( + session=session, + projects=projects, + only_active=only_active, + prev_created_at=prev_created_at, + prev_id=prev_id, + limit=limit, + ascending=ascending, + ) + return [endpoint_model_to_endpoint(e) for e in endpoint_models] + + +async def list_projects_endpoint_models( + session: AsyncSession, + projects: List[ProjectModel], + only_active: bool, + prev_created_at: Optional[datetime], + prev_id: Optional[uuid.UUID], + limit: int, + ascending: bool, +) -> List[EndpointModel]: + if not projects: + return [] + filters: list[Any] = [EndpointModel.project_id.in_(p.id for p in projects)] + if only_active: + filters.append(EndpointModel.deleted == False) + if prev_created_at is not None: + if ascending: + if prev_id is None: + filters.append(EndpointModel.created_at > prev_created_at) + else: + filters.append( + or_( + EndpointModel.created_at > prev_created_at, + and_( + EndpointModel.created_at == prev_created_at, + EndpointModel.id < prev_id, + ), + ) + ) + else: + if prev_id is None: + filters.append(EndpointModel.created_at < prev_created_at) + else: + filters.append( + or_( + EndpointModel.created_at < prev_created_at, + and_( + EndpointModel.created_at == prev_created_at, + EndpointModel.id > prev_id, + ), + ) + ) + order_by = (EndpointModel.created_at.desc(), EndpointModel.id) + if ascending: + order_by = (EndpointModel.created_at.asc(), EndpointModel.id.desc()) + res = await session.execute( + select(EndpointModel) + .where(*filters) + .order_by(*order_by) + .limit(limit) + .options(joinedload(EndpointModel.user)) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.service_run)) + ) + return list(res.unique().scalars().all()) + + +async def list_project_endpoints( + session: AsyncSession, + project: ProjectModel, + names: Optional[List[str]] = None, +) -> List[Endpoint]: + endpoint_models = await list_project_endpoint_models( + session=session, project=project, names=names + ) + return [endpoint_model_to_endpoint(e) for e in endpoint_models] + + +async def list_project_endpoint_models( + session: AsyncSession, + project: ProjectModel, + names: Optional[List[str]] = None, + include_deleted: bool = False, +) -> List[EndpointModel]: + filters = [EndpointModel.project_id == project.id] + if names is not None: + filters.append(EndpointModel.name.in_(names)) + if not include_deleted: + filters.append(EndpointModel.deleted == False) + res = await session.execute( + select(EndpointModel) + .where(*filters) + .options(joinedload(EndpointModel.user)) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.service_run)) + ) + return list(res.unique().scalars().all()) + + +async def get_endpoint_by_name( + session: AsyncSession, project: ProjectModel, name: str +) -> Optional[Endpoint]: + endpoint_model = await get_project_endpoint_model_by_name( + session=session, project=project, name=name + ) + if endpoint_model is None: + return None + return endpoint_model_to_endpoint(endpoint_model) + + +async def get_project_endpoint_model_by_name( + session: AsyncSession, + project: ProjectModel, + name: str, + include_deleted: bool = False, +) -> Optional[EndpointModel]: + filters = [ + EndpointModel.name == name, + EndpointModel.project_id == project.id, + ] + if not include_deleted: + filters.append(EndpointModel.deleted == False) + res = await session.execute( + select(EndpointModel) + .where(*filters) + .options(joinedload(EndpointModel.user)) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.service_run)) + ) + return res.unique().scalar_one_or_none() + + +async def get_endpoint_plan( + session: AsyncSession, + project: ProjectModel, + user: UserModel, + configuration: EndpointConfiguration, + configuration_path: Optional[str], +) -> EndpointPlan: + current_resource = None + if configuration.name is not None: + current_resource = await get_endpoint_by_name( + session=session, + project=project, + name=configuration.name, + ) + if current_resource is not None and current_resource.status.is_finished(): + current_resource = None + preset_plan = None + unprovisionable_preset_plan = None + if configuration.preset_policy != EndpointPresetPolicy.CREATE: + preset_planning_result = await find_preset_planning_result( + session=session, + project=project, + user=user, + endpoint_name=configuration.name, + endpoint_configuration=configuration, + ) + preset_plan = preset_planning_result.provisionable + unprovisionable_preset_plan = preset_planning_result.unprovisionable + if preset_plan is None and configuration.preset_policy == EndpointPresetPolicy.REUSE: + preset_plan = unprovisionable_preset_plan + provisioning_plan = _get_no_provisioning_plan( + configuration.preset_policy, + unprovisionable_preset_plan=unprovisionable_preset_plan, + ) + if preset_plan is not None: + provisioning_plan = _endpoint_preset_plan_to_provisioning_plan(preset_plan) + elif ( + configuration.preset_policy != EndpointPresetPolicy.REUSE + and get_agent_service().is_enabled() + ): + provisioning_plan = _agent_plan_to_provisioning_plan( + get_agent_service().get_plan(), + max_budget=get_effective_max_agent_budget(configuration), + reason=_get_unprovisionable_preset_reason(unprovisionable_preset_plan), + ) + return EndpointPlan( + project_name=project.name, + user=user.name, + configuration=configuration, + configuration_path=configuration_path, + current_resource=current_resource, + action=ApplyAction.CREATE if current_resource is None else ApplyAction.UPDATE, + preset_policy=configuration.preset_policy, + provisioning_plan=provisioning_plan, + ) + + +def _agent_plan_to_provisioning_plan( + agent_plan: AgentPlan, + max_budget: Optional[float], + reason: Optional[str] = None, +) -> EndpointProvisioningPlanAgent: + return EndpointProvisioningPlanAgent( + agent_model=agent_plan.model, + max_budget=max_budget, + reason=reason, + ) + + +def _get_no_provisioning_plan( + preset_policy: EndpointPresetPolicy, + unprovisionable_preset_plan: Optional[EndpointPresetPlan] = None, +) -> EndpointProvisioningPlanNone: + if preset_policy == EndpointPresetPolicy.REUSE: + reason = _get_unprovisionable_preset_reason(unprovisionable_preset_plan) + if reason is None: + reason = "No matching endpoint presets found." + return EndpointProvisioningPlanNone(reason=reason) + agent_unavailable_reason = get_agent_unavailable_reason() + preset_reason = _get_unprovisionable_preset_reason(unprovisionable_preset_plan) + if preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: + if preset_reason is not None: + return EndpointProvisioningPlanNone( + reason=( + f"{preset_reason} Creating a preset requires the server agent, " + f"but {agent_unavailable_reason}" + ) + ) + return EndpointProvisioningPlanNone( + reason=( + "No matching endpoint presets found. Creating a preset requires the server " + f"agent, but {agent_unavailable_reason}" + ) + ) + return EndpointProvisioningPlanNone( + reason=(f"Preset policy create requires the server agent, but {agent_unavailable_reason}") + ) + + +def _get_unprovisionable_preset_reason( + preset_plan: Optional[EndpointPresetPlan], +) -> Optional[str]: + if preset_plan is None: + return None + return f"Endpoint preset {preset_plan.preset.name} matched but has no available offers." + + +def _endpoint_preset_plan_to_provisioning_plan( + preset_plan: EndpointPresetPlan, +) -> EndpointProvisioningPlanPreset: + run_spec = preset_plan.run_plan.get_effective_run_spec() + service_name = run_spec.run_name or run_spec.configuration.name or "(generated)" + return EndpointProvisioningPlanPreset( + preset_name=preset_plan.preset.name, + service_name=service_name, + replica_spec_groups=[ + EndpointPlanReplicaSpecGroup( + name=group.name, + replica_specs=group.replica_specs, + ) + for group in preset_plan.preset.replica_spec_groups + ], + job_offers=[ + EndpointPlanJobOffers( + replica_group=job_plan.job_spec.replica_group, + resources=job_plan.job_spec.requirements.resources, + spot=job_plan.job_spec.requirements.spot, + max_price=job_plan.job_spec.requirements.max_price, + offers=job_plan.offers, + total_offers=job_plan.total_offers, + max_offer_price=job_plan.max_price, + ) + for job_plan in preset_plan.run_plan.job_plans + ], + ) + + +async def create_endpoint( + session: AsyncSession, + project: ProjectModel, + user: UserModel, + configuration: EndpointConfiguration, + pipeline_hinter: PipelineHinterProtocol, +) -> Endpoint: + _validate_endpoint_configuration(configuration) + + lock_namespace = f"endpoint_names_{project.name}" + if is_db_sqlite(): + await session.commit() + elif is_db_postgres(): + await session.execute( + select(func.pg_advisory_xact_lock(string_to_lock_id(lock_namespace))) + ) + lock, _ = get_locker(get_db().dialect_name).get_lockset(lock_namespace) + async with lock: + now = common.get_current_datetime() + if configuration.name is not None: + endpoint_model = await get_project_endpoint_model_by_name( + session=session, + project=project, + name=configuration.name, + ) + if endpoint_model is not None: + if not endpoint_model.status.is_finished(): + raise ResourceExistsError() + endpoint_model.deleted = True + endpoint_model.deleted_at = now + else: + configuration.name = await generate_endpoint_name(session=session, project=project) + + endpoint_model = EndpointModel( + id=uuid.uuid4(), + name=configuration.name, + project=project, + user_id=user.id, + status=EndpointStatus.SUBMITTED, + configuration=configuration.json(), + created_at=now, + last_processed_at=now, + ) + session.add(endpoint_model) + events.emit( + session, + message=f"Endpoint created. Status: {endpoint_model.status.upper()}", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(endpoint_model)], + ) + await session.commit() + pipeline_hinter.hint_fetch(EndpointModel.__name__) + return endpoint_model_to_endpoint(endpoint_model) + + +async def delete_endpoints( + session: AsyncSession, + project: ProjectModel, + names: List[str], + user: UserModel, + pipeline_hinter: Optional[PipelineHinterProtocol] = None, +): + endpoint_models = await list_project_endpoint_models( + session=session, + project=project, + names=names, + ) + now = common.get_current_datetime() + for endpoint_model in endpoint_models: + if endpoint_model.to_be_deleted: + continue + endpoint_model.to_be_deleted = True + endpoint_model.deletion_requested_at = now + events.emit( + session, + message="Endpoint marked for deletion", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(endpoint_model)], + ) + await session.commit() + if pipeline_hinter is not None: + pipeline_hinter.hint_fetch(EndpointModel.__name__) + + +def endpoint_model_to_endpoint(endpoint_model: EndpointModel) -> Endpoint: + configuration = get_endpoint_configuration(endpoint_model) + run_name = None + url = None + status = endpoint_model.status + if status == EndpointStatus.ACTIVE: + status = EndpointStatus.RUNNING + if endpoint_model.service_run is not None and not endpoint_model.service_run.deleted: + run_name = endpoint_model.service_run.run_name + if endpoint_model.service_run.service_spec is not None: + service_spec = ServiceSpec.__response__.parse_raw( + endpoint_model.service_run.service_spec + ) + if service_spec.model is not None: + url = service_spec.model.base_url + return Endpoint( + id=endpoint_model.id, + name=endpoint_model.name, + project_name=endpoint_model.project.name, + user=endpoint_model.user.name, + configuration=configuration, + created_at=endpoint_model.created_at, + last_processed_at=endpoint_model.last_processed_at, + status=status, + status_message=endpoint_model.status_message, + deleted=endpoint_model.deleted, + deleted_at=endpoint_model.deleted_at, + run_name=run_name, + url=url, + error=endpoint_model.status_message if status == EndpointStatus.FAILED else None, + ) + + +def get_endpoint_configuration(endpoint_model: EndpointModel) -> EndpointConfiguration: + return EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) + + +async def record_endpoint_run_submission( + session: AsyncSession, + endpoint_id: uuid.UUID, + run_id: uuid.UUID, +) -> EndpointRunSubmissionModel: + existing_submission = await _get_endpoint_run_submission_by_run_id( + session=session, + run_id=run_id, + ) + if existing_submission is not None: + if existing_submission.endpoint_id != endpoint_id: + raise ServerClientError("Run is already recorded for another endpoint") + return existing_submission + + res = await session.execute( + select(func.max(EndpointRunSubmissionModel.submission_num)).where( + EndpointRunSubmissionModel.endpoint_id == endpoint_id + ) + ) + submission_num = (res.scalar_one_or_none() or 0) + 1 + submission = EndpointRunSubmissionModel( + endpoint_id=endpoint_id, + run_id=run_id, + submission_num=submission_num, + submitted_at=common.get_current_datetime(), + ) + session.add(submission) + await session.flush() + return submission + + +async def _get_endpoint_run_submission_by_run_id( + session: AsyncSession, + run_id: uuid.UUID, +) -> Optional[EndpointRunSubmissionModel]: + res = await session.execute( + select(EndpointRunSubmissionModel).where(EndpointRunSubmissionModel.run_id == run_id) + ) + return res.scalar_one_or_none() + + +async def generate_endpoint_name(session: AsyncSession, project: ProjectModel) -> str: + res = await session.execute( + select(EndpointModel.name).where( + EndpointModel.project_id == project.id, + EndpointModel.deleted == False, + ) + ) + names = set(res.scalars().all()) + while True: + name = random_names.generate_name() + if name not in names: + return name + + +def _validate_endpoint_configuration(configuration: EndpointConfiguration): + if not configuration.model.strip(): + raise ServerClientError("Endpoint must specify model") + if configuration.name is not None: + validate_dstack_resource_name(configuration.name) + try: + configuration.env.as_dict() + except ValueError as e: + raise ServerClientError(f"Endpoint env is unresolved: {e}") from e diff --git a/src/dstack/_internal/server/services/endpoints/agent/__init__.py b/src/dstack/_internal/server/services/endpoints/agent/__init__.py new file mode 100644 index 0000000000..68510cd8f6 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/__init__.py @@ -0,0 +1,91 @@ +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Optional + +from dstack._internal.core.models.endpoints import EndpointConfiguration +from dstack._internal.server import settings +from dstack._internal.server.models import EndpointModel +from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport +from dstack._internal.server.services.pipelines import PipelineHinterProtocol + + +@dataclass(frozen=True) +class AgentPlan: + model: str + + +@dataclass(frozen=True) +class AgentProvisioningResult: + run_id: Optional[uuid.UUID] = None + run_name: Optional[str] = None + error: Optional[str] = None + final_report: Optional[AgentFinalReport] = None + + +class AgentService(ABC): + @abstractmethod + def is_enabled(self) -> bool: + pass + + @abstractmethod + def get_plan(self) -> AgentPlan: + pass + + @abstractmethod + async def provision_endpoint( + self, + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, + ) -> AgentProvisioningResult: + pass + + +class DisabledAgentService(AgentService): + def __init__(self, reason: Optional[str] = None) -> None: + self._reason = reason + + def is_enabled(self) -> bool: + return False + + def get_plan(self) -> AgentPlan: + return AgentPlan(model=settings.AGENT_ANTHROPIC_MODEL) + + async def provision_endpoint( + self, + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, + ) -> AgentProvisioningResult: + return AgentProvisioningResult(error=self._reason or get_agent_unavailable_reason()) + + +def get_agent_service() -> AgentService: + if settings.AGENT_ANTHROPIC_API_KEY: + from dstack._internal.server.services.endpoints.agent.claude import ( + ClaudeAgentService, + get_claude_agent_unavailable_reason, + ) + + unavailable_reason = get_claude_agent_unavailable_reason() + if unavailable_reason is None: + return ClaudeAgentService() + return DisabledAgentService(reason=unavailable_reason) + return DisabledAgentService(reason=get_agent_unavailable_reason()) + + +def get_agent_unavailable_reason() -> Optional[str]: + if not settings.AGENT_ANTHROPIC_API_KEY: + return "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + from dstack._internal.server.services.endpoints.agent.claude import ( + get_claude_agent_unavailable_reason, + ) + + return get_claude_agent_unavailable_reason() + + +def get_effective_max_agent_budget( + configuration: EndpointConfiguration, +) -> Optional[float]: + if configuration.max_agent_budget is not None: + return configuration.max_agent_budget + return settings.AGENT_ANTHROPIC_MAX_BUDGET diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py new file mode 100644 index 0000000000..290862d220 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -0,0 +1,977 @@ +import asyncio +import json +import os +import shutil +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, Optional, Sequence +from uuid import UUID + +import yaml +from pydantic import ValidationError + +from dstack._internal.core.models.config import GlobalConfig, ProjectConfig +from dstack._internal.core.models.endpoints import EndpointConfiguration +from dstack._internal.server import settings +from dstack._internal.server.models import EndpointModel, ProjectModel +from dstack._internal.server.schemas.runner import LogEvent +from dstack._internal.server.services import logs as logs_services +from dstack._internal.server.services.endpoints.agent import ( + AgentPlan, + AgentProvisioningResult, + AgentService, + get_effective_max_agent_budget, +) +from dstack._internal.server.services.endpoints.agent.report import ( + AGENT_FINAL_REPORT_JSON_SCHEMA, + AgentFinalReport, +) +from dstack._internal.server.services.pipelines import PipelineHinterProtocol +from dstack._internal.utils.common import get_milliseconds_since_epoch, run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +_RESOURCE_DIR = Path(__file__).parent / "resources" +_PROMPT_RESOURCE_NAMES = [ + "system_prompt.md", + "dstack_cli_and_service_authoring.md", + "recipes_guide.md", + "deployment_harness.md", +] +_INHERITED_ENV_NAMES = [ + "PATH", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] +_REDACTION = "[redacted]" +_MAX_CAPTURED_OUTPUT_CHARS = 20_000 +_MAX_AGENT_LOG_MESSAGE_CHARS = 4_000 +_MAX_TOOL_RESULT_LOG_PREVIEW_CHARS = 1_200 +_MAX_TOOL_RESULT_LOG_PREVIEW_LINES = 30 +_AGENT_LOG_BATCH_SIZE = 1 +_CLAUDE_AGENT_TOOLS = "Bash,Read,Write,Edit,WebFetch,WebSearch,StructuredOutput" + + +@dataclass(frozen=True) +class _AgentRunnerResult: + report: Optional[AgentFinalReport] = None + error: Optional[str] = None + + +@dataclass +class _AgentProcessOutput: + report_data: Optional[dict[str, Any]] = None + result_error: Optional[str] = None + stdout_tail: str = "" + + +class ClaudeAgentService(AgentService): + def __init__( + self, + runner: Optional[ + Callable[["_AgentWorkspace", dict[str, Any]], Awaitable[_AgentRunnerResult]] + ] = None, + workspace_base_dir: Path = settings.SERVER_DATA_DIR_PATH / "endpoint_agent_runs", + ) -> None: + self._runner = runner or _run_agent_in_subprocess + self._workspace_base_dir = workspace_base_dir + + def is_enabled(self) -> bool: + return get_claude_agent_unavailable_reason() is None + + def get_plan(self) -> AgentPlan: + return AgentPlan(model=settings.AGENT_ANTHROPIC_MODEL) + + async def provision_endpoint( + self, + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, + ) -> AgentProvisioningResult: + unavailable_reason = get_claude_agent_unavailable_reason() + if unavailable_reason is not None: + return AgentProvisioningResult(error=unavailable_reason) + + try: + workspace = _prepare_workspace( + endpoint_model=endpoint_model, + workspace_base_dir=self._workspace_base_dir, + ) + except Exception as e: + logger.warning("Failed to prepare endpoint agent workspace: %s", e, exc_info=True) + return AgentProvisioningResult( + error=f"Failed to prepare endpoint agent workspace: {e}" + ) + + runner_result = await self._runner(workspace, _build_agent_request(workspace)) + if runner_result.error is not None: + workspace.artifacts.record_error(runner_result.error) + return AgentProvisioningResult(error=runner_result.error) + report = runner_result.report + if report is None: + workspace.artifacts.record_error("Server agent did not return a verification report") + return AgentProvisioningResult( + error="Server agent did not return a verification report" + ) + workspace.artifacts.record_report(report) + + logger.info( + "Endpoint agent finished for endpoint %s: success=%s run_id=%s run_name=%s", + endpoint_model.name, + report.success, + report.run_id, + report.run_name, + ) + return AgentProvisioningResult( + run_id=report.run_id, + run_name=report.run_name, + final_report=report, + ) + + +def get_claude_agent_unavailable_reason() -> Optional[str]: + if not settings.AGENT_ANTHROPIC_API_KEY: + return "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + if _get_claude_executable() is None: + if settings.AGENT_CLAUDE_PATH is not None: + return ( + "DSTACK_AGENT_CLAUDE_PATH does not resolve to an executable: " + f"{settings.AGENT_CLAUDE_PATH}" + ) + return ( + "server agent runtime requires the claude executable in PATH or " + "DSTACK_AGENT_CLAUDE_PATH." + ) + return None + + +class _AgentWorkspace: + def __init__( + self, + *, + root_dir: Path, + home_dir: Path, + work_dir: Path, + trace_path: Optional[Path], + env: dict[str, str], + redacted_values: Sequence[str], + endpoint_name: str, + model: str, + max_agent_budget: Optional[float], + endpoint_constraints: str = "", + max_price: Optional[float] = None, + spot_policy: Optional[str] = None, + log_writer: Optional["_AgentLogWriter"] = None, + ) -> None: + self.root_dir = root_dir + self.home_dir = home_dir + self.work_dir = work_dir + self.trace_path = trace_path + self.env = env + self.redacted_values = tuple(redacted_values) + self.endpoint_name = endpoint_name + self.model = model + self.max_agent_budget = max_agent_budget + self.endpoint_constraints = endpoint_constraints + self.max_price = max_price + self.spot_policy = spot_policy + self.log_writer = log_writer + self.artifacts = _AgentArtifactRecorder(self) + + +class _AgentLogWriter: + def __init__( + self, + *, + project: ProjectModel, + endpoint_id: UUID, + endpoint_name: str, + ) -> None: + self._project = project + self._endpoint_id = endpoint_id + self._endpoint_name = endpoint_name + self._buffer: list[LogEvent] = [] + + async def write(self, message: str) -> None: + message = _truncate_log_message(message) + if not message.endswith("\n"): + message += "\n" + self._buffer.append( + LogEvent( + timestamp=get_milliseconds_since_epoch(), + message=message.encode(), + ) + ) + if len(self._buffer) >= _AGENT_LOG_BATCH_SIZE: + await self.flush() + + async def flush(self) -> None: + if not self._buffer: + return + events = self._buffer + self._buffer = [] + try: + await run_async( + logs_services.write_logs, + project=self._project, + run_name=self._endpoint_name, + job_submission_id=self._endpoint_id, + runner_logs=[], + job_logs=events, + ) + except Exception: + logger.warning( + "Failed to write endpoint agent logs for endpoint %s", + self._endpoint_name, + exc_info=True, + ) + + +def _prepare_workspace( + *, + endpoint_model: EndpointModel, + workspace_base_dir: Path, +) -> _AgentWorkspace: + configuration = EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) + + root_dir = workspace_base_dir / str(endpoint_model.id) + home_dir = root_dir / "home" + work_dir = root_dir / "workspace" + dstack_dir = home_dir / ".dstack" + work_dir.mkdir(parents=True, exist_ok=True) + dstack_dir.mkdir(parents=True, exist_ok=True) + token = endpoint_model.user.token.get_plaintext_or_error() + _write_cli_config( + dstack_dir=dstack_dir, + project_name=endpoint_model.project.name, + server_url=settings.SERVER_URL, + token=token, + ) + + endpoint_env = configuration.env.as_dict() + env = _build_agent_env( + home_dir=home_dir, + project_name=endpoint_model.project.name, + endpoint_env=endpoint_env, + ) + redacted_values = _get_redacted_values( + [ + token, + settings.AGENT_ANTHROPIC_API_KEY or "", + *endpoint_env.values(), + *_get_sensitive_inherited_env_values(env), + ] + ) + trace_path = root_dir / "trace.jsonl" if _is_debug_trace_enabled() else None + workspace = _AgentWorkspace( + root_dir=root_dir, + home_dir=home_dir, + work_dir=work_dir, + trace_path=trace_path, + env=env, + redacted_values=redacted_values, + endpoint_name=endpoint_model.name, + model=configuration.model, + max_agent_budget=get_effective_max_agent_budget(configuration), + endpoint_constraints=_format_endpoint_constraints(configuration, endpoint_env), + max_price=configuration.max_price, + spot_policy=configuration.spot_policy.value if configuration.spot_policy else None, + log_writer=_AgentLogWriter( + project=endpoint_model.project, + endpoint_id=endpoint_model.id, + endpoint_name=endpoint_model.name, + ), + ) + workspace.artifacts.initialize() + return workspace + + +class _AgentArtifactRecorder: + def __init__(self, workspace: _AgentWorkspace) -> None: + self._workspace = workspace + self._command_counter = 0 + + def initialize(self) -> None: + self._workspace.work_dir.mkdir(parents=True, exist_ok=True) + self._update_agent_state(phase="starting") + for filename in ["sources.jsonl", "candidates.jsonl", "commands.jsonl"]: + (self._workspace.work_dir / filename).touch(exist_ok=True) + hardware_reasoning_path = self._workspace.work_dir / "hardware_reasoning.md" + if not hardware_reasoning_path.exists(): + hardware_reasoning_path.write_text( + ( + "# Hardware Reasoning\n\n" + "The endpoint agent should update this file with the model size,\n" + "serving framework, resource estimate, offer/fleet choice, and\n" + "why the selected hardware is credible.\n" + ), + encoding="utf-8", + ) + + def mark_running(self) -> None: + self._update_agent_state(phase="running") + + def record_report(self, report: AgentFinalReport) -> None: + (self._workspace.work_dir / "final_report.json").write_text( + json.dumps( + _redact(report.dict(), self._workspace.redacted_values), + default=str, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + self._update_agent_state(phase="success" if report.success else "failure") + + def record_error(self, error: str) -> None: + data = { + "success": False, + "failure_summary": error, + "recorded_at": _utcnow_iso(), + } + (self._workspace.work_dir / "agent_error.json").write_text( + json.dumps(_redact(data, self._workspace.redacted_values), indent=2) + "\n", + encoding="utf-8", + ) + self._update_agent_state(phase="failure") + + def record_stream_message(self, message: dict[str, Any]) -> None: + self._record_bash_tool_uses(message) + self._record_tool_results(message) + + def _update_agent_state(self, phase: str) -> None: + path = self._workspace.work_dir / "agent_state.json" + state: dict[str, Any] = {} + if path.exists(): + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + parsed = {} + if isinstance(parsed, dict): + state.update(parsed) + now = _utcnow_iso() + state.update( + { + "endpoint_name": self._workspace.endpoint_name, + "model": self._workspace.model, + "phase": phase, + "max_agent_budget": self._workspace.max_agent_budget, + "max_hourly_price": self._workspace.max_price, + "spot_policy": self._workspace.spot_policy, + "updated_at": now, + } + ) + state.setdefault("started_at", now) + path.write_text( + json.dumps(_redact(state, self._workspace.redacted_values), indent=2) + "\n", + encoding="utf-8", + ) + + def _record_bash_tool_uses(self, message: dict[str, Any]) -> None: + if message.get("type") != "assistant": + return + claude_message = message.get("message") + if not isinstance(claude_message, dict): + return + for item in claude_message.get("content", []): + if not isinstance(item, dict): + continue + if item.get("type") != "tool_use" or item.get("name") != "Bash": + continue + tool_input = item.get("input") + if not isinstance(tool_input, dict): + continue + command = tool_input.get("command") + if not isinstance(command, str) or not command.strip(): + continue + record = { + "event": "tool_use", + "timestamp": _utcnow_iso(), + "tool_use_id": item.get("id"), + "description": tool_input.get("description"), + "command": command, + } + self._append_jsonl("commands.jsonl", record) + + def _record_tool_results(self, message: dict[str, Any]) -> None: + if message.get("type") != "user": + return + claude_message = message.get("message") + if not isinstance(claude_message, dict): + return + for item in claude_message.get("content", []): + if not isinstance(item, dict) or item.get("type") != "tool_result": + continue + content = item.get("content") + output_path = None + if isinstance(content, str) and content.strip(): + output_path = self._write_command_output(content) + record = { + "event": "tool_result", + "timestamp": _utcnow_iso(), + "tool_use_id": item.get("tool_use_id") or message.get("parent_tool_use_id"), + "is_error": bool(item.get("is_error")), + "output_path": output_path, + } + self._append_jsonl("commands.jsonl", record) + + def _write_command_output(self, content: str) -> str: + self._command_counter += 1 + output_dir = self._workspace.work_dir / "command-output" + output_dir.mkdir(parents=True, exist_ok=True) + relative_path = Path("command-output") / f"{self._command_counter:04d}.txt" + output_path = self._workspace.work_dir / relative_path + output_path.write_text( + _redact(content, self._workspace.redacted_values), + encoding="utf-8", + ) + return relative_path.as_posix() + + def _append_jsonl(self, filename: str, record: dict[str, Any]) -> None: + path = self._workspace.work_dir / filename + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write( + json.dumps(_redact(record, self._workspace.redacted_values), default=str) + "\n" + ) + + +def _write_cli_config( + *, + dstack_dir: Path, + project_name: str, + server_url: str, + token: str, +) -> None: + config = GlobalConfig( + projects=[ + ProjectConfig( + name=project_name, + url=server_url, + token=token, + default=True, + ) + ] + ) + (dstack_dir / "config.yml").write_text( + yaml.safe_dump(json.loads(config.json()), sort_keys=False), + encoding="utf-8", + ) + + +def _build_agent_env( + *, + home_dir: Path, + project_name: str, + endpoint_env: dict[str, str], +) -> dict[str, str]: + env = {name: value for name in _INHERITED_ENV_NAMES if (value := os.environ.get(name))} + env.update( + { + "ANTHROPIC_API_KEY": settings.AGENT_ANTHROPIC_API_KEY or "", + "HOME": str(home_dir), + "DSTACK_PROJECT": project_name, + } + ) + env.update(endpoint_env) + return env + + +def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: + return { + "prompt": _build_prompt(workspace), + "cwd": str(workspace.work_dir), + "env": workspace.env, + "trace_path": str(workspace.trace_path) if workspace.trace_path is not None else None, + "redacted_values": list(workspace.redacted_values), + "options": { + "tools": _CLAUDE_AGENT_TOOLS, + "allowed_tools": _CLAUDE_AGENT_TOOLS, + "disallowed_tools": "Task,NotebookEdit", + "model": settings.AGENT_ANTHROPIC_MODEL, + "max_turns": None, + "max_budget": workspace.max_agent_budget, + "json_schema": AGENT_FINAL_REPORT_JSON_SCHEMA, + }, + } + + +def _build_prompt(workspace: _AgentWorkspace) -> str: + return f"""Deploy endpoint {workspace.endpoint_name!r} for model {workspace.model!r}. + +Use the real `dstack` CLI available in this process. Do not use any custom dstack APIs +or wait for any server-side helper tool. The embedded guidance below is the dstack and +dstack-development playbook for this run. + +{workspace.endpoint_constraints} + +Required final service: +- service YAML type: service +- service YAML must set `model: {workspace.model}` +- submit runs detached with `dstack apply -f -y -d` +- use concise, unique run names that are useful for debugging +- the successful final report must include the verified service run `run_id` + +{_load_prompt_resources()} + +When you have a terminal result, write `final_report.json` in the workspace with the +same fields requested by the JSON schema, then return only the structured final report. +""" + + +def _load_prompt_resources() -> str: + return "\n\n".join( + (_RESOURCE_DIR / resource_name).read_text(encoding="utf-8").strip() + for resource_name in _PROMPT_RESOURCE_NAMES + ) + + +def _format_endpoint_constraints( + configuration: EndpointConfiguration, + endpoint_env: dict[str, str], +) -> str: + lines = [ + "Binding endpoint constraints:", + "- Every `dstack offer`, `dstack apply` preview, and submitted service must honor these constraints.", + "- Do not submit a service if the plan violates these constraints.", + ] + cli_flags = [] + if configuration.max_price is not None: + lines.append(f"- max_price: {configuration.max_price}") + cli_flags.extend(["--max-price", str(configuration.max_price)]) + if configuration.spot_policy is not None: + lines.append(f"- spot_policy: {configuration.spot_policy.value}") + if configuration.spot_policy.value == "spot": + cli_flags.append("--spot") + elif configuration.spot_policy.value == "on-demand": + cli_flags.append("--on-demand") + else: + cli_flags.append("--spot-auto") + for field, flag in [ + ("backends", "--backend"), + ("regions", "--region"), + ("instance_types", "--instance-type"), + ("fleets", "--fleet"), + ]: + values = getattr(configuration, field) + if not values: + continue + formatted_values = [_format_constraint_value(value) for value in values] + lines.append(f"- {field}: {', '.join(formatted_values)}") + for value in formatted_values: + cli_flags.extend([flag, value]) + for field in ["availability_zones", "instances"]: + values = getattr(configuration, field) + if not values: + continue + formatted_values = [_format_constraint_value(value) for value in values] + lines.append(f"- {field}: {', '.join(formatted_values)}") + if configuration.creation_policy is not None: + lines.append(f"- creation_policy: {configuration.creation_policy.value}") + if configuration.creation_policy.value == "reuse": + cli_flags.append("--reuse") + if cli_flags: + lines.append(f"- Reuse these CLI flags where applicable: {' '.join(cli_flags)}") + else: + lines.append("- No explicit profile constraints were set.") + if endpoint_env: + lines.append( + f"- Endpoint env keys available in the agent environment: {', '.join(endpoint_env)}" + ) + else: + lines.append("- Endpoint env keys: none") + return "\n".join(lines) + + +def _format_constraint_value(value: Any) -> str: + if hasattr(value, "name") and getattr(value, "project", None) is None: + return str(value.name) + if hasattr(value, "json"): + return json.dumps(json.loads(value.json()), sort_keys=True) + if hasattr(value, "value"): + return str(value.value) + return str(value) + + +async def _run_agent_in_subprocess( + workspace: _AgentWorkspace, + request: dict[str, Any], +) -> _AgentRunnerResult: + cmd = _build_claude_command(request) + await _write_agent_log( + workspace, + _format_agent_start_log(request), + ) + workspace.artifacts.mark_running() + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=workspace.work_dir, + env=request["env"], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert proc.stdout is not None + assert proc.stderr is not None + stdout_task = asyncio.create_task(_read_agent_stdout(proc.stdout, workspace)) + stderr_task = asyncio.create_task(_read_agent_stderr(proc.stderr, workspace)) + stdout_output, stderr_output, returncode = await asyncio.gather( + stdout_task, + stderr_task, + proc.wait(), + ) + process_output = _merge_process_outputs(stdout_output, stderr_output) + _write_trace_record( + workspace, + { + "type": "agent-process", + "returncode": returncode, + }, + ) + await _write_agent_log(workspace, f"Server agent process exited with code {returncode}") + await _flush_agent_logs(workspace) + if process_output.report_data is None: + process_output.report_data = _load_final_report_artifact(workspace) + if process_output.report_data is None: + if process_output.result_error is not None: + return _AgentRunnerResult( + error="Server agent failed before returning a verification report" + ) + if returncode not in (0, None): + return _AgentRunnerResult( + error=( + "Server agent process exited without a verification report " + f"(return code {returncode})" + ) + ) + return _AgentRunnerResult( + error="Server agent process exited without a verification report" + ) + try: + return _AgentRunnerResult(report=AgentFinalReport.parse_obj(process_output.report_data)) + except ValidationError as e: + return _AgentRunnerResult( + error=f"Server agent returned an invalid verification report: {e}" + ) + + +def _build_claude_command(request: dict[str, Any]) -> list[str]: + options = request["options"] + cmd = [ + _get_claude_executable() or "claude", + "-p", + "--bare", + "--output-format", + "stream-json", + "--verbose", + "--tools", + options.get("tools", "default"), + "--allowedTools", + options["allowed_tools"], + "--disallowedTools", + options["disallowed_tools"], + "--permission-mode", + "bypassPermissions", + "--model", + options["model"], + "--json-schema", + json.dumps(options["json_schema"]), + request["prompt"], + ] + if options["max_turns"] is not None: + cmd[2:2] = ["--max-turns", str(options["max_turns"])] + if options["max_budget"] is not None: + cmd[2:2] = ["--max-budget-usd", str(options["max_budget"])] + return cmd + + +def _get_claude_executable() -> Optional[str]: + if settings.AGENT_CLAUDE_PATH is not None: + return shutil.which(settings.AGENT_CLAUDE_PATH) + return shutil.which("claude") + + +async def _read_agent_stdout( + stream: asyncio.StreamReader, + workspace: _AgentWorkspace, +) -> _AgentProcessOutput: + return await _read_agent_stream(stream, workspace, stream_name="stdout") + + +async def _read_agent_stderr( + stream: asyncio.StreamReader, + workspace: _AgentWorkspace, +) -> _AgentProcessOutput: + return await _read_agent_stream(stream, workspace, stream_name="stderr") + + +async def _read_agent_stream( + stream: asyncio.StreamReader, + workspace: _AgentWorkspace, + stream_name: str, +) -> _AgentProcessOutput: + output = _AgentProcessOutput() + while True: + line_bytes = await stream.readline() + if not line_bytes: + return output + line = line_bytes.decode(errors="replace") + output.stdout_tail = _append_bounded_output(output.stdout_tail, line) + if not line.strip(): + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + message = {"type": "raw-output", "stream": stream_name, "line": line.rstrip("\r\n")} + _write_trace_record(workspace, message) + await _write_agent_log_if_present(workspace, message) + continue + if stream_name != "stdout": + message.setdefault("stream", stream_name) + _write_trace_record(workspace, message) + workspace.artifacts.record_stream_message(message) + _update_agent_process_output(output, message) + await _write_agent_log_if_present(workspace, message) + + +def _update_agent_process_output( + output: _AgentProcessOutput, + message: dict[str, Any], +) -> None: + if message.get("type") != "result": + return + if message.get("is_error"): + output.result_error = message.get("result") or "Claude agent failed" + structured_output = message.get("structured_output") + if isinstance(structured_output, dict): + output.report_data = structured_output + return + report_data = _parse_report_json(message.get("result")) + if report_data is not None: + output.report_data = report_data + + +def _append_bounded_output(existing: str, addition: str) -> str: + combined = existing + addition + if len(combined) <= _MAX_CAPTURED_OUTPUT_CHARS: + return combined + return combined[-_MAX_CAPTURED_OUTPUT_CHARS:] + + +def _merge_process_outputs(*outputs: _AgentProcessOutput) -> _AgentProcessOutput: + merged = _AgentProcessOutput() + for output in outputs: + if output.report_data is not None: + merged.report_data = output.report_data + if output.result_error is not None: + merged.result_error = output.result_error + merged.stdout_tail = _append_bounded_output(merged.stdout_tail, output.stdout_tail) + return merged + + +async def _write_agent_log_if_present( + workspace: _AgentWorkspace, + message: dict[str, Any], +) -> None: + log_message = _format_agent_log_message(message) + if log_message is not None: + await _write_agent_log(workspace, log_message) + + +async def _write_agent_log(workspace: _AgentWorkspace, message: str) -> None: + if workspace.log_writer is None: + return + message = _redact(message, workspace.redacted_values) + await workspace.log_writer.write(message) + + +async def _flush_agent_logs(workspace: _AgentWorkspace) -> None: + if workspace.log_writer is None: + return + await workspace.log_writer.flush() + + +def _format_agent_start_log(request: dict[str, Any]) -> str: + options = request["options"] + parts = [f"Starting server agent ({options['model']})"] + if options["max_budget"] is not None: + parts.append(f"max budget ${options['max_budget']}") + cwd = request.get("cwd") + if cwd: + parts.append(f"workspace {cwd}") + return ", ".join(parts) + + +def _format_agent_log_message(message: dict[str, Any]) -> Optional[str]: + message_type = message.get("type") + if message_type == "system" and message.get("subtype") == "init": + model = message.get("model") + version = message.get("claude_code_version") + details = ", ".join( + str(v) for v in [model, f"Claude Code {version}" if version else None] if v + ) + return f"Server agent initialized ({details})" if details else "Server agent initialized" + if message_type == "assistant": + return _format_assistant_message_log(message.get("message")) + if message_type == "user": + return _format_user_message_log(message.get("message")) + if message_type == "result": + if message.get("is_error"): + result = message.get("result") + if isinstance(result, str) and result.strip(): + return f"Server agent failed: {_truncate_log_message(result.strip())}" + return "Server agent failed" + return "Server agent finished" + if message_type == "raw-output": + line = message.get("line") + if isinstance(line, str) and line.strip(): + prefix = "Agent stderr" if message.get("stream") == "stderr" else "Agent output" + return f"{prefix}: {line.strip()}" + return None + + +def _format_assistant_message_log(message: Any) -> Optional[str]: + if not isinstance(message, dict): + return None + lines = [] + for item in message.get("content", []): + if not isinstance(item, dict): + continue + if item.get("type") == "tool_use": + name = item.get("name") or "tool" + tool_input = item.get("input") + description = None + command = None + if isinstance(tool_input, dict): + description = tool_input.get("description") + command = tool_input.get("command") + if isinstance(description, str) and description.strip(): + lines.append(f"Agent tool: {name} ({description.strip()})") + else: + lines.append(f"Agent tool: {name}") + if isinstance(command, str) and command.strip(): + lines.append(f"$ {command.strip()}") + elif item.get("type") == "text" and isinstance(item.get("text"), str): + text = item["text"].strip() + if text: + lines.append(text) + if not lines: + return None + return "\n".join(lines) + + +def _format_user_message_log(message: Any) -> Optional[str]: + if not isinstance(message, dict): + return None + lines = [] + for item in message.get("content", []): + if not isinstance(item, dict) or item.get("type") != "tool_result": + continue + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + continue + prefix = "Tool error" if item.get("is_error") else "Tool output" + lines.append(_format_tool_result_log(prefix, content)) + if not lines: + return None + return "\n\n".join(lines) + + +def _format_tool_result_log(prefix: str, content: str) -> str: + stripped = content.strip() + result_lines = stripped.splitlines() + if ( + len(stripped) <= _MAX_TOOL_RESULT_LOG_PREVIEW_CHARS + and len(result_lines) <= _MAX_TOOL_RESULT_LOG_PREVIEW_LINES + ): + return f"{prefix}:\n{stripped}" + + preview_lines = result_lines[:_MAX_TOOL_RESULT_LOG_PREVIEW_LINES] + preview = "\n".join(preview_lines) + if len(preview) > _MAX_TOOL_RESULT_LOG_PREVIEW_CHARS: + preview = preview[: _MAX_TOOL_RESULT_LOG_PREVIEW_CHARS - 15].rstrip() + if preview: + preview += "\n... truncated" + else: + preview = "... truncated" + return ( + f"{prefix} captured ({len(stripped)} chars, {len(result_lines)} lines). " + "Full output is stored in the endpoint agent workspace.\n" + f"{preview}" + ) + + +def _truncate_log_message(message: str) -> str: + if len(message) <= _MAX_AGENT_LOG_MESSAGE_CHARS: + return message + return message[: _MAX_AGENT_LOG_MESSAGE_CHARS - 15].rstrip() + "\n... truncated" + + +def _parse_report_json(value: Any) -> Optional[dict[str, Any]]: + if not isinstance(value, str): + return None + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + return parsed + + +def _load_final_report_artifact(workspace: _AgentWorkspace) -> Optional[dict[str, Any]]: + path = workspace.work_dir / "final_report.json" + if not path.exists(): + return None + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + logger.warning("Endpoint agent final_report.json is not valid JSON: %s", path) + return None + if not isinstance(parsed, dict): + logger.warning("Endpoint agent final_report.json is not an object: %s", path) + return None + return parsed + + +def _write_trace_record(workspace: _AgentWorkspace, data: dict[str, Any]) -> None: + if workspace.trace_path is None: + return + workspace.trace_path.parent.mkdir(parents=True, exist_ok=True) + line = json.dumps(_redact(data, workspace.redacted_values), default=str) + with workspace.trace_path.open("a", encoding="utf-8") as f: + f.write(line + "\n") + + +def _get_redacted_values(values: Sequence[str]) -> tuple[str, ...]: + return tuple(sorted({value for value in values if value}, key=len, reverse=True)) + + +def _get_sensitive_inherited_env_values(env: dict[str, str]) -> list[str]: + return [env[name] for name in _INHERITED_ENV_NAMES if name != "PATH" and name in env] + + +def _redact(value: Any, redacted_values: Sequence[str]) -> Any: + if isinstance(value, str): + for redacted_value in redacted_values: + value = value.replace(redacted_value, _REDACTION) + return value + if isinstance(value, dict): + return {k: _redact(v, redacted_values) for k, v in value.items()} + if isinstance(value, list): + return [_redact(item, redacted_values) for item in value] + return value + + +def _is_debug_trace_enabled() -> bool: + return settings.LOG_LEVEL == "DEBUG" or settings.ROOT_LOG_LEVEL == "DEBUG" + + +def _utcnow_iso() -> str: + return datetime.now(timezone.utc).isoformat() diff --git a/src/dstack/_internal/server/services/endpoints/agent/report.py b/src/dstack/_internal/server/services/endpoints/agent/report.py new file mode 100644 index 0000000000..ed872d6b4f --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/report.py @@ -0,0 +1,48 @@ +import uuid +from typing import Optional + +from pydantic import Field, root_validator + +from dstack._internal.core.models.common import CoreModel + +AGENT_FINAL_REPORT_JSON_SCHEMA = { + "type": "object", + "properties": { + "success": {"type": "boolean"}, + "run_id": {"type": "string"}, + "run_name": {"type": "string"}, + "service_yaml": {"type": "string"}, + "recipe_sources": {"type": "array", "items": {"type": "string"}}, + "verification_summary": {"type": "string"}, + "failure_summary": {"type": "string"}, + }, + "required": ["success"], + "additionalProperties": False, +} + + +class AgentFinalReport(CoreModel): + # TODO: Keep this contract intentionally minimal until real agent runs show + # which validation evidence is actually useful and reliable. + success: bool + run_id: Optional[uuid.UUID] = None + run_name: Optional[str] = None + service_yaml: Optional[str] = None + recipe_sources: list[str] = Field(default_factory=list) + verification_summary: Optional[str] = None + failure_summary: Optional[str] = None + + @root_validator + def _validate_report(cls, values: dict) -> dict: + success = values.get("success") + if success: + if values.get("run_id") is None: + raise ValueError("successful agent report must include run_id") + if not values.get("service_yaml"): + raise ValueError("successful agent report must include service_yaml") + if not values.get("verification_summary"): + raise ValueError("successful agent report must include verification_summary") + else: + if not values.get("failure_summary"): + raise ValueError("failed agent report must include failure_summary") + return values diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md new file mode 100644 index 0000000000..9867a0b790 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md @@ -0,0 +1,42 @@ +# Deployment Harness + +Use this lifecycle: + +1. Understand the model: source URLs, serving framework options, expected memory/disk, and any model-specific launch requirements. +2. Inspect project capacity: use `dstack offer` and/or existing fleets with the endpoint's binding constraints. +3. Pick the fastest credible experiment path: + - If evidence is strong, preview and submit the final service candidate directly. + - If startup commands, images, framework versions, or hardware are uncertain, prototype first. + - Prototypes may be dstack services, tasks, or dev environments when that is the most effective way to test commands interactively. +4. Keep one active candidate at a time where possible. Stop bad candidates before replacing them. +5. Watch status using JSON run state, events, and logs. Use short probes and update workspace artifacts; do not hide progress inside long `sleep` or `until` loops. +6. When the final service reaches `running`, check `dstack run get --json` for job status and probe state. A useful readiness signal is a running job with an HTTP probe `success_streak` greater than zero. Do not wait for a `registered` field if it is absent or null. +7. Get the model URL from `dstack run get --json`, combine it with the configured server URL from `~/.dstack/config.yml` if it is relative, and send a real OpenAI-compatible chat request for the requested model. +8. Once the model request succeeds, write `verification.json`, write `final_report.json`, and return the structured final report immediately. Otherwise return a failed report with the useful evidence and the last candidate state. + +Workspace artifact contract: + +- On startup or resume, inspect `agent_state.json`, `candidates.jsonl`, `verification.json`, and `final_report.json` before submitting anything new. If a previous candidate is already running and verifiable, reuse it instead of creating a duplicate run. +- Keep `agent_state.json` current enough to show the phase. +- Add source evidence to `sources.jsonl`. +- Keep `hardware_reasoning.md` reviewable before any paid run. +- Record every spend-capable service, task, or dev-environment candidate in `candidates.jsonl`. +- The server records shell command/tool output in `commands.jsonl` and `command-output/`; use those artifacts when diagnosing. +- On success, write `verification.json` with the final model request evidence before returning the final report. +- Always write `final_report.json` before returning the structured final report. + +Development environments and tasks are allowed for experimentation. Use them when they reduce uncertainty, for example to test an image, install path, model download, launch command, or framework compatibility before committing to the final service. They are not the endpoint: the final verified run reported to the server must be a dstack service. + +For v1, do not attempt P/D disaggregation, router/worker multi-service topologies, autoscaling tuning, load benchmarking, or performance optimization unless they are strictly necessary to make the requested model serve at all. Note when those would be the right next step. + +Hardware behavior: + +- Do not blindly select the cheapest offer. +- Prefer hardware likely to run the serving image reliably: enough VRAM, enough disk, common CUDA-capable NVIDIA GPUs when using CUDA images, and offers without obvious provisioning instability. +- If a candidate stays in backend provisioning without logs/events progress after several polls, inspect run JSON/events and any available native backend or SSH/TCP evidence, then stop or fail with evidence rather than looping forever. +- If a backend or dstack provisioning issue is found, create or reference a minimal non-endpoint reproduction in `endpoint-agent-backend-troubleshooting.md` when possible. + +Final report: + +- On success, include the final service run id, final service run name, the exact final service YAML, recipe/source URLs, and a concise verification summary. +- On failure, include the failure summary and enough evidence for the next iteration to improve the harness. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md b/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md new file mode 100644 index 0000000000..b39628e252 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md @@ -0,0 +1,25 @@ +# dstack CLI and Service Authoring + +Core CLI rules: + +- Preview before submit with `printf 'n\n' | dstack apply -f ...`. +- Submit runs detached with `dstack apply -f -y -d`. +- Never run `dstack apply` for a run without `-d` in this non-interactive server context. +- Do not use `dstack ps` table text for machine decisions; table output can be truncated. Use `dstack run get --json` for machine decisions. +- Use `dstack event --within-run ` and `dstack logs ` to understand provisioning, pulling, startup, and runtime failures. +- A run can be `running` before the in-server proxy can answer. Before probing the service URL, check `dstack run get --json` for a running job and HTTP probe state. If a probe has `success_streak > 0`, proceed to a real model request. Do not block on a `registered` field if it is absent or null. +- If a run is externally stopped or has `termination_reason: stopped_by_user`, do not resubmit it. +- Stop a bad candidate before replacing it: `dstack stop -y`. +- Retry only after changing the underlying hypothesis, YAML, command, image, hardware, or constraints. + +Service essentials: + +- The final endpoint-backed YAML must be `type: service`; choose a concise, unique run name that is useful for debugging. +- The final service YAML must include `model: ` so dstack exposes a model URL. +- Services usually need `port: 8000` and commands that start an OpenAI-compatible server. +- Common starting points are `vllm serve ` and `python -m sglang.launch_server --model-path --host 0.0.0.0 --port 8000`, but verify current docs and model-specific notes before trusting these defaults. +- Pass endpoint environment variables by name, for example `env: [HF_TOKEN]`; never write secret values into YAML or logs. +- Choose `resources` from evidence: model size, framework requirements, GPU memory, disk needed for weights/cache, and current dstack offers/fleets. +- For OpenAI-compatible verification, use `service.model.base_url` from `dstack run get --json` and POST to `/chat/completions` with the requested model. If the base URL is relative, prefix it with the default project URL from `~/.dstack/config.yml`. + +Use dstack plans as real constraints. If the endpoint supplied `max_price`, `spot_policy`, backend, region, instance type, fleet, or reuse constraints, every offer lookup, preview, and submitted experiment must honor them. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md b/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md new file mode 100644 index 0000000000..1c706f7c04 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md @@ -0,0 +1,24 @@ +# Recipe and Hardware Grounding + +Before choosing a deployment, gather enough evidence to explain why the service command and hardware should work. + +Primary sources to prefer: + +- vLLM docs and recipes: https://docs.vllm.ai/ and https://recipes.vllm.ai/models.json +- SGLang docs and cookbook: https://docs.sglang.ai/ +- Hugging Face model cards and framework notes, for example https://huggingface.co/docs/transformers/model_doc/qwen3 and https://huggingface.co/Qwen/Qwen3-0.6B +- dstack documentation and local CLI help. + +Supporting sources for advanced direction, not v1 requirements: + +- https://www.wafer.ai/blog/glm52-amd +- https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development/ +- https://github.com/dstackai/dstack/pull/3856 + +Recipe selection rubric: + +- Prefer an official framework recipe for the exact model. If none exists, use the model family recipe plus the model card. +- Check whether the model needs `trust_remote_code`, special tokenizer/chat template handling, quantization, tensor parallelism, or specific framework versions. +- Estimate VRAM and disk before looking at offers. For a tiny model, avoid overbuying unless the cheaper path is unstable. For larger models, do not guess; use model size, precision, quantization, KV-cache, and tensor-parallel evidence. +- When evidence is uncertain, use a bounded experiment instead of guessing. +- Record recipe/source URLs in the final report so a learned preset has provenance. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md new file mode 100644 index 0000000000..8737d8aa7d --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md @@ -0,0 +1,13 @@ +# Objective + +You are the server-side endpoint deployment agent for dstack. Your job is to turn an endpoint request into a working, verified dstack service for the requested model. + +This is a real deployment investigation, not a YAML-generation task. Use the local workspace to keep notes, service/dev/task YAML files, command transcripts, backend observations, and evidence. Use the real `dstack` CLI and shell commands directly. Verify CLI flags with `dstack --help` when unsure. + +Do not invent dstack flags or YAML properties. Do not call hidden server APIs. Do not wait for custom helper functions such as `find_model_recipes`, `submit_service`, or `get_run_logs`; use normal files, shell commands, web sources, and the `dstack` CLI. + +You may use network research through the available web tools and shell commands. Prefer primary, current sources and preserve URLs in the final report. Treat model cards, official serving docs, recipe indexes, and successful command/log evidence as stronger than generic snippets. + +The final endpoint may only be reported as successful after the final dstack service run is running, exposes the model endpoint, and has answered a real model request for the requested model. + +If verification succeeds, stop investigating and produce the final report immediately. Do not continue optimizing, benchmarking, or trying alternate hardware after a correct working service is proven. diff --git a/src/dstack/_internal/server/services/endpoints/names.py b/src/dstack/_internal/server/services/endpoints/names.py new file mode 100644 index 0000000000..7552fc9022 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/names.py @@ -0,0 +1,12 @@ +from typing import Optional + +_DSTACK_RESOURCE_NAME_MAX_LENGTH = 41 +_SERVING_RUN_SUFFIX = "-serving" + + +def get_endpoint_serving_run_name(endpoint_name: Optional[str]) -> Optional[str]: + if endpoint_name is None: + return None + if len(endpoint_name) + len(_SERVING_RUN_SUFFIX) <= _DSTACK_RESOURCE_NAME_MAX_LENGTH: + return f"{endpoint_name}{_SERVING_RUN_SUFFIX}" + return endpoint_name diff --git a/src/dstack/_internal/server/services/endpoints/planning.py b/src/dstack/_internal/server/services/endpoints/planning.py new file mode 100644 index 0000000000..e2926145e8 --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/planning.py @@ -0,0 +1,170 @@ +from dataclasses import dataclass +from typing import Optional, Sequence + +from sqlalchemy.ext.asyncio import AsyncSession + +from dstack._internal.core.errors import ServerClientError +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.endpoints import EndpointConfiguration +from dstack._internal.core.models.envs import EnvSentinel +from dstack._internal.core.models.profiles import ProfileParams +from dstack._internal.core.models.runs import JobPlan, RunPlan, RunSpec +from dstack._internal.server.models import ProjectModel, UserModel +from dstack._internal.server.services import runs as runs_services +from dstack._internal.server.services import users as users_services +from dstack._internal.server.services.endpoints.names import get_endpoint_serving_run_name +from dstack._internal.server.services.endpoints.presets import ( + EndpointPreset, + EndpointPresetService, + get_endpoint_preset_service, +) +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class EndpointPresetPlan: + """A preset selected by endpoint planning and the run plan computed from it.""" + + preset: EndpointPreset + run_plan: RunPlan + + +@dataclass(frozen=True) +class EndpointPresetPlanningResult: + """Preset planning result split into provisionable and no-offer matches.""" + + provisionable: Optional[EndpointPresetPlan] = None + unprovisionable: Optional[EndpointPresetPlan] = None + + +async def find_matching_preset_plan( + session: AsyncSession, + project: ProjectModel, + user: UserModel, + endpoint_name: Optional[str], + endpoint_configuration: EndpointConfiguration, + max_offers: int = 1, + preset_service: Optional[EndpointPresetService] = None, +) -> Optional[EndpointPresetPlan]: + result = await find_preset_planning_result( + session=session, + project=project, + user=user, + endpoint_name=endpoint_name, + endpoint_configuration=endpoint_configuration, + max_offers=max_offers, + preset_service=preset_service, + ) + return result.provisionable + + +async def find_preset_planning_result( + session: AsyncSession, + project: ProjectModel, + user: UserModel, + endpoint_name: Optional[str], + endpoint_configuration: EndpointConfiguration, + max_offers: int = 1, + preset_service: Optional[EndpointPresetService] = None, +) -> EndpointPresetPlanningResult: + if preset_service is None: + preset_service = get_endpoint_preset_service() + + endpoint_model = endpoint_configuration.model.lower() + presets = [ + preset + for preset in await preset_service.list_presets() + if preset.model.lower() == endpoint_model + ] + if not presets: + return EndpointPresetPlanningResult() + + user = await _ensure_user_has_ssh_key(session=session, user=user) + first_unprovisionable_preset: Optional[EndpointPresetPlan] = None + for preset in presets: + try: + run_spec = build_preset_run_spec( + endpoint_name=endpoint_name, + endpoint_configuration=endpoint_configuration, + preset=preset, + ) + _validate_run_spec_env_resolved(run_spec) + run_plan = await runs_services.get_plan( + session=session, + project=project, + user=user, + run_spec=run_spec, + max_offers=max_offers, + ) + except (ServerClientError, ValueError) as e: + logger.warning("Skipping endpoint preset %s: %s", preset.name, e) + continue + preset_plan = EndpointPresetPlan(preset=preset, run_plan=run_plan) + if _run_plan_has_available_offers(run_plan.job_plans): + return EndpointPresetPlanningResult( + provisionable=preset_plan, + unprovisionable=first_unprovisionable_preset, + ) + if first_unprovisionable_preset is None: + first_unprovisionable_preset = preset_plan + return EndpointPresetPlanningResult(unprovisionable=first_unprovisionable_preset) + + +def build_preset_service_configuration( + endpoint_name: Optional[str], + endpoint_configuration: EndpointConfiguration, + preset: EndpointPreset, +) -> ServiceConfiguration: + service_configuration = preset.configuration.copy(deep=True) + service_configuration.name = get_endpoint_serving_run_name(endpoint_name) + service_configuration.env.update(endpoint_configuration.env) + for field in ProfileParams.__fields__: + value = getattr(endpoint_configuration, field) + if value is not None: + setattr(service_configuration, field, value) + return service_configuration + + +def build_preset_run_spec( + endpoint_name: Optional[str], + endpoint_configuration: EndpointConfiguration, + preset: EndpointPreset, +) -> RunSpec: + service_configuration = build_preset_service_configuration( + endpoint_name=endpoint_name, + endpoint_configuration=endpoint_configuration, + preset=preset, + ) + return RunSpec( + run_name=service_configuration.name, + configuration=service_configuration, + ) + + +def _validate_run_spec_env_resolved(run_spec: RunSpec) -> None: + unresolved = [ + key for key, value in run_spec.configuration.env.items() if isinstance(value, EnvSentinel) + ] + if unresolved: + raise ValueError("preset env is unresolved: " + ", ".join(sorted(unresolved))) + + +async def _ensure_user_has_ssh_key( + session: AsyncSession, + user: UserModel, +) -> UserModel: + if user.ssh_public_key: + return user + refreshed_user = await users_services.refresh_ssh_key(session=session, actor=user) + if refreshed_user is None: + return user + return refreshed_user + + +def _run_plan_has_available_offers(job_plans: Sequence[JobPlan]) -> bool: + return bool(job_plans) and all( + any(offer.availability.is_available() for offer in job_plan.offers) + for job_plan in job_plans + ) diff --git a/src/dstack/_internal/server/services/endpoints/preset_building.py b/src/dstack/_internal/server/services/endpoints/preset_building.py new file mode 100644 index 0000000000..6c3860590a --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/preset_building.py @@ -0,0 +1,110 @@ +from collections import defaultdict +from typing import Any + +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.instances import InstanceOfferWithAvailability, Resources +from dstack._internal.core.models.resources import ResourcesSpec +from dstack._internal.core.models.runs import JobStatus +from dstack._internal.server.models import JobModel, RunModel +from dstack._internal.server.services import jobs as jobs_services +from dstack._internal.server.services import runs as runs_services +from dstack._internal.server.services.endpoints.presets import ( + EndpointPreset, + EndpointPresetReplicaSpecGroup, +) +from dstack._internal.utils.common import format_mib_as_gb + + +def build_endpoint_preset_from_run(name: str, run_model: RunModel) -> EndpointPreset: + run_spec = runs_services.get_run_spec(run_model) + configuration = run_spec.configuration + if not isinstance(configuration, ServiceConfiguration): + raise ValueError("endpoint preset can only be built from a service run") + if configuration.model is None: + raise ValueError("endpoint preset service must specify model") + + jobs_by_group = _get_current_registered_replica_jobs_by_group(run_model) + replica_spec_groups = [] + for replica_group in configuration.replica_groups: + group_name = replica_group.name + if group_name is None: + raise ValueError("endpoint preset cannot be built from unnamed replica groups") + group_jobs = jobs_by_group.get(group_name, []) + if not group_jobs: + raise ValueError( + f"endpoint preset cannot be built: replica group {group_name!r} " + "has no registered running replicas" + ) + replica_spec_groups.append( + EndpointPresetReplicaSpecGroup( + name=group_name, + replica_specs=[_get_job_resources_spec(job) for job in group_jobs], + ) + ) + + return EndpointPreset( + name=name, + model=configuration.model.name, + replica_spec_groups=replica_spec_groups, + configuration=configuration.copy(deep=True), + ) + + +def _get_current_registered_replica_jobs_by_group( + run_model: RunModel, +) -> dict[str, list[JobModel]]: + jobs_by_group = defaultdict(list) + for job in sorted(run_model.jobs, key=lambda j: (j.replica_num, j.job_num)): + if job.deployment_num != run_model.deployment_num: + continue + if job.job_num != 0: + continue + if job.status != JobStatus.RUNNING or not job.registered: + continue + job_spec = jobs_services.get_job_spec(job) + jobs_by_group[job_spec.replica_group].append(job) + return jobs_by_group + + +def _get_job_resources_spec(job_model: JobModel) -> ResourcesSpec: + offer = _get_job_offer(job_model) + if offer is not None: + return _resources_spec_from_instance_resources(offer.instance.resources) + return jobs_services.get_job_spec(job_model).requirements.resources + + +def _get_job_offer(job_model: JobModel) -> InstanceOfferWithAvailability | None: + job_runtime_data = jobs_services.get_job_runtime_data(job_model) + if job_runtime_data is not None and job_runtime_data.offer is not None: + return job_runtime_data.offer + instance = job_model.__dict__.get("instance") + if instance is not None and instance.offer is not None: + return InstanceOfferWithAvailability.__response__.parse_raw(instance.offer) + return None + + +def _resources_spec_from_instance_resources(resources: Resources) -> ResourcesSpec: + data: dict[str, Any] = { + "cpu": str(resources.cpus), + "memory": format_mib_as_gb(resources.memory_mib), + "disk": format_mib_as_gb(resources.disk.size_mib), + } + if resources.cpu_arch is not None: + data["cpu"] = f"{resources.cpu_arch.value}:{resources.cpus}" + if resources.gpus: + first_gpu = resources.gpus[0] + if any( + gpu.name != first_gpu.name + or gpu.memory_mib != first_gpu.memory_mib + or gpu.vendor != first_gpu.vendor + for gpu in resources.gpus + ): + raise ValueError("endpoint preset cannot be built from mixed-GPU instances") + data["gpu"] = { + "name": first_gpu.name, + "memory": format_mib_as_gb(first_gpu.memory_mib), + "count": len(resources.gpus), + } + if first_gpu.vendor is not None: + data["gpu"]["vendor"] = first_gpu.vendor.value + return ResourcesSpec.parse_obj(data) diff --git a/src/dstack/_internal/server/services/endpoints/presets.py b/src/dstack/_internal/server/services/endpoints/presets.py new file mode 100644 index 0000000000..ec86d4ec3f --- /dev/null +++ b/src/dstack/_internal/server/services/endpoints/presets.py @@ -0,0 +1,339 @@ +import json +import os +import re +import uuid +from abc import ABC, abstractmethod +from copy import deepcopy +from pathlib import Path +from typing import Any, Optional, Sequence + +import yaml +from pydantic import ValidationError + +from dstack._internal.core.models.common import CoreModel +from dstack._internal.core.models.configurations import ( + DEFAULT_REPLICA_GROUP_NAME, + ServiceConfiguration, +) +from dstack._internal.core.models.envs import EnvSentinel +from dstack._internal.core.models.profiles import ProfileParams +from dstack._internal.core.models.resources import ResourcesSpec +from dstack._internal.server import settings +from dstack._internal.utils.common import run_async +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +_SECRET_ENV_PATTERN = re.compile(r"(token|key|secret|password)", re.IGNORECASE) + + +class EndpointPresetReplicaSpecGroup(CoreModel): + """Ordered to match `ServiceConfiguration.replica_groups`; "0" is the implicit group.""" + + name: str + replica_specs: list[ResourcesSpec] + + +class EndpointPreset(CoreModel): + name: str + model: str + replica_spec_groups: list[EndpointPresetReplicaSpecGroup] + """Sorted to match `configuration.replica_groups`; use group name "0" for the implicit group.""" + configuration: ServiceConfiguration + + +class EndpointPresetService(ABC): + @abstractmethod + async def list_presets(self) -> list[EndpointPreset]: + pass + + @abstractmethod + async def save_preset( + self, + preset: EndpointPreset, + comments: Optional[Sequence[str]] = None, + ) -> EndpointPreset: + """Store a prepared preset and return it with the storage-assigned name.""" + pass + + +class LocalDirEndpointPresetService(EndpointPresetService): + def __init__(self, presets_dir: Path = settings.ENDPOINT_PRESETS_DIR) -> None: + self._presets_dir = presets_dir + + async def list_presets(self) -> list[EndpointPreset]: + return await run_async(self._list_presets) + + async def save_preset( + self, + preset: EndpointPreset, + comments: Optional[Sequence[str]] = None, + ) -> EndpointPreset: + return await run_async(self._save_preset, preset, comments or []) + + def _list_presets(self) -> list[EndpointPreset]: + if not self._presets_dir.exists(): + return [] + presets = [] + for path in sorted(self._presets_dir.iterdir()): + if path.suffix not in [".yml", ".yaml"]: + continue + preset = self._load_preset(path) + if preset is not None: + presets.append(preset) + return presets + + def _load_preset(self, path: Path) -> Optional[EndpointPreset]: + try: + with path.open("r") as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise ValueError("preset must be a YAML object") + if data.get("type") != "endpoint-preset": + raise ValueError("preset must be an endpoint preset") + model = data.get("model") + if not isinstance(model, str) or model == "": + raise ValueError("preset must specify a model") + service_data = _get_service_data(data) + replica_spec_groups = _get_preset_replica_spec_groups(data, service_data) + service_data.setdefault("model", model) + service_data["type"] = "service" + configuration = ServiceConfiguration.parse_obj(service_data) + if configuration.model is None: + raise ValueError("preset service configuration must specify model") + if configuration.model.name.lower() != model.lower(): + raise ValueError("preset model must match the service model") + return EndpointPreset( + name=_get_preset_name_from_path(path), + model=model, + replica_spec_groups=replica_spec_groups, + configuration=configuration, + ) + except (OSError, ValidationError, ValueError, yaml.YAMLError) as e: + logger.warning("Skipping endpoint preset %s: %s", path, e) + return None + + def _save_preset(self, preset: EndpointPreset, comments: Sequence[str]) -> EndpointPreset: + self._presets_dir.mkdir(parents=True, exist_ok=True) + data = _preset_to_data(preset) + content = _format_preset_comments(comments) + yaml.safe_dump(data, sort_keys=False) + base_name = _slugify_preset_name(preset.name) + suffix = 0 + while True: + name = base_name if suffix == 0 else f"{base_name}-{suffix + 1}" + path = self._presets_dir / f"{name}.dstack.yml" + try: + self._write_preset_file(path=path, content=content) + break + except FileExistsError: + suffix += 1 + saved_preset = self._load_preset(path) + if saved_preset is None: + raise ValueError(f"saved endpoint preset {path} could not be loaded") + return saved_preset + + def _write_preset_file(self, path: Path, content: str) -> None: + tmp_path = self._presets_dir / f".{path.name}.{uuid.uuid4().hex}.tmp" + tmp_path.write_text(content, encoding="utf-8") + try: + os.link(tmp_path, path) + finally: + tmp_path.unlink(missing_ok=True) + + +_endpoint_preset_service: EndpointPresetService = LocalDirEndpointPresetService() + + +def get_endpoint_preset_service() -> EndpointPresetService: + return _endpoint_preset_service + + +def _get_preset_name_from_path(path: Path) -> str: + name = path.stem + if name.endswith(".dstack"): + name = name[: -len(".dstack")] + return name + + +def _get_service_data(data: dict[str, Any]) -> dict[str, Any]: + service_data = data.get("service") + if not isinstance(service_data, dict): + raise ValueError("preset must specify a service object") + if service_data.get("type") not in (None, "service"): + raise ValueError("preset service object must be a service configuration") + if "name" in service_data: + raise ValueError("preset service object must not specify name") + if "resources" in service_data: + raise ValueError("preset service object must not specify resources") + profile_fields = sorted(set(ProfileParams.__fields__) & set(service_data)) + if profile_fields: + raise ValueError( + "preset service object must not specify profile fields: " + ", ".join(profile_fields) + ) + _validate_service_replica_groups(service_data) + return deepcopy(service_data) + + +def _preset_to_data(preset: EndpointPreset) -> dict[str, Any]: + return { + "type": "endpoint-preset", + "model": preset.model, + "service": _service_configuration_to_preset_data(preset.configuration), + "replica_spec_groups": [ + _replica_spec_group_to_data(group) for group in preset.replica_spec_groups + ], + } + + +def _service_configuration_to_preset_data( + configuration: ServiceConfiguration, +) -> dict[str, Any]: + service_data = json.loads(configuration.json(exclude_none=True)) + service_data.pop("type", None) + service_data.pop("name", None) + service_data.pop("resources", None) + for field in ProfileParams.__fields__: + service_data.pop(field, None) + if configuration.env: + service_data["env"] = [ + _env_item_to_preset_data(key, value) + for key, value in sorted(configuration.env.items()) + ] + else: + service_data.pop("env", None) + for field, value in list(service_data.items()): + if value in ({}, []): + service_data.pop(field) + replicas = service_data.get("replicas") + if isinstance(replicas, list): + for group in replicas: + if isinstance(group, dict): + group.pop("resources", None) + return service_data + + +def _replica_spec_group_to_data(group: EndpointPresetReplicaSpecGroup) -> dict[str, Any]: + return { + "name": group.name, + "replica_specs": [ + json.loads(replica_spec.json(exclude_none=True)) + for replica_spec in group.replica_specs + ], + } + + +def _env_item_to_preset_data(key: str, value: str | EnvSentinel) -> str: + if isinstance(value, EnvSentinel) or _is_secret_like_env(key=key, value=value): + return key + return f"{key}={value}" + + +def _is_secret_like_env(key: str, value: str | EnvSentinel) -> bool: + if _SECRET_ENV_PATTERN.search(key): + return True + return isinstance(value, str) and _SECRET_ENV_PATTERN.search(value) is not None + + +def _format_preset_comments(comments: Sequence[str]) -> str: + if not comments: + return "" + lines = [] + for comment in comments: + for line in comment.splitlines() or [""]: + lines.append(f"# {line}".rstrip()) + return "\n".join(lines) + "\n" + + +def _slugify_preset_name(name: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + return slug or "endpoint-preset" + + +def _validate_service_replica_groups(service_data: dict[str, Any]) -> None: + replicas = service_data.get("replicas") + if not isinstance(replicas, list): + return + for group in replicas: + if not isinstance(group, dict): + raise ValueError("preset service replica groups must be objects") + if "resources" in group: + raise ValueError("preset service replica groups must not specify resources") + + +def _get_preset_replica_spec_groups( + data: dict[str, Any], + service_data: dict[str, Any], +) -> list[EndpointPresetReplicaSpecGroup]: + if "resources" in data or "replica_resources" in data: + raise ValueError("preset must specify replica_spec_groups") + raw_replica_spec_groups = data.get("replica_spec_groups") + if not isinstance(raw_replica_spec_groups, list) or not raw_replica_spec_groups: + raise ValueError("preset must specify non-empty replica_spec_groups") + if not all(isinstance(group, dict) for group in raw_replica_spec_groups): + raise ValueError("preset replica_spec_groups must be a list of objects") + raw_replica_spec_groups = [dict(group) for group in raw_replica_spec_groups] + expected_names = _get_expected_replica_group_names(service_data) + if expected_names == [DEFAULT_REPLICA_GROUP_NAME] and "name" not in raw_replica_spec_groups[0]: + raw_replica_spec_groups[0]["name"] = DEFAULT_REPLICA_GROUP_NAME + replica_spec_groups = [ + EndpointPresetReplicaSpecGroup.parse_obj(group) for group in raw_replica_spec_groups + ] + _validate_replica_spec_groups( + replica_spec_groups=replica_spec_groups, expected_names=expected_names + ) + _apply_replica_spec_group_resources( + service_data=service_data, + replica_spec_groups=replica_spec_groups, + ) + return replica_spec_groups + + +def _get_expected_replica_group_names(service_data: dict[str, Any]) -> list[str]: + replicas = service_data.get("replicas") + if not isinstance(replicas, list): + return [DEFAULT_REPLICA_GROUP_NAME] + return [_get_replica_group_name(group) for group in replicas] + + +def _get_replica_group_name(replica_group: Any) -> str: + if not isinstance(replica_group, dict): + raise ValueError("preset service replica groups must be objects") + name = replica_group.get("name") + if not isinstance(name, str) or name == "": + raise ValueError( + "preset service replica groups must specify names when using replica_spec_groups" + ) + return name + + +def _validate_replica_spec_groups( + replica_spec_groups: list[EndpointPresetReplicaSpecGroup], + expected_names: list[str], +) -> None: + group_names = [group.name for group in replica_spec_groups] + if group_names != expected_names: + raise ValueError( + "preset replica_spec_groups must match replica group order: " + + ", ".join(expected_names) + ) + for group in replica_spec_groups: + if not group.replica_specs: + raise ValueError("preset replica_spec_groups must specify non-empty replica_specs") + first_resources = group.replica_specs[0].dict() + if any(resources.dict() != first_resources for resources in group.replica_specs): + raise ValueError("preset replica_specs within one group must have the same resources") + + +def _apply_replica_spec_group_resources( + service_data: dict[str, Any], + replica_spec_groups: list[EndpointPresetReplicaSpecGroup], +) -> None: + replicas = service_data.get("replicas") + if not isinstance(replicas, list): + service_data["resources"] = replica_spec_groups[0].replica_specs[0].dict() + return + resources_by_group = { + group.name: group.replica_specs[0].dict() for group in replica_spec_groups + } + for group in replicas: + group["resources"] = resources_by_group[group["name"]] diff --git a/src/dstack/_internal/server/services/events.py b/src/dstack/_internal/server/services/events.py index dd7b33dc7f..960fa4379f 100644 --- a/src/dstack/_internal/server/services/events.py +++ b/src/dstack/_internal/server/services/events.py @@ -11,6 +11,7 @@ from dstack._internal.core.models.users import GlobalRole from dstack._internal.server import settings from dstack._internal.server.models import ( + EndpointModel, EventModel, EventTargetModel, FleetModel, @@ -97,6 +98,7 @@ def from_model( SecretModel, UserModel, VolumeModel, + EndpointModel, ], ) -> "Target": if isinstance(model, FleetModel): @@ -162,6 +164,13 @@ def from_model( id=model.id, name=model.name, ) + if isinstance(model, EndpointModel): + return Target( + type=EventTargetType.ENDPOINT, + project_id=model.project_id or model.project.id, + id=model.id, + name=model.name, + ) raise ValueError(f"Unsupported model type: {type(model)}") def fmt(self) -> str: diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 27a97a6db5..218c141580 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -12,6 +12,21 @@ logger = get_logger(__name__) + +def _parse_positive_float(value: str) -> float: + parsed = float(value) + if parsed <= 0: + raise ValueError("value must be greater than 0") + return parsed + + +def _getenv_non_empty(name: str) -> str | None: + value = os.getenv(name) + if value is None or value == "": + return None + return value + + DSTACK_DIR_PATH = Path("~/.dstack/").expanduser() SERVER_DIR_PATH = Path(os.getenv("DSTACK_SERVER_DIR", DSTACK_DIR_PATH / "server")).resolve() @@ -21,6 +36,17 @@ SERVER_DATA_DIR_PATH = SERVER_DIR_PATH / "data" SERVER_DATA_DIR_PATH.mkdir(parents=True, exist_ok=True) +ENDPOINT_PRESETS_DIR = Path( + os.getenv("DSTACK_SERVER_ENDPOINT_PRESETS_DIR", SERVER_DATA_DIR_PATH / "endpoint_presets") +).resolve() + +AGENT_ANTHROPIC_API_KEY = _getenv_non_empty("DSTACK_AGENT_ANTHROPIC_API_KEY") +AGENT_CLAUDE_PATH = _getenv_non_empty("DSTACK_AGENT_CLAUDE_PATH") +AGENT_ANTHROPIC_MODEL = os.getenv("DSTACK_AGENT_ANTHROPIC_MODEL", "claude-opus-4-8") +AGENT_ANTHROPIC_MAX_BUDGET = environ.get_callback( + "DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD", _parse_positive_float +) + DATABASE_URL = os.getenv( "DSTACK_DATABASE_URL", f"sqlite+aiosqlite:///{str(SERVER_DATA_DIR_PATH.absolute())}/sqlite.db" ) diff --git a/src/dstack/_internal/server/testing/common.py b/src/dstack/_internal/server/testing/common.py index 3b24ef96e5..142faac84c 100644 --- a/src/dstack/_internal/server/testing/common.py +++ b/src/dstack/_internal/server/testing/common.py @@ -438,12 +438,18 @@ async def create_job( disconnected_at: Optional[datetime] = None, registered: bool = False, waiting_master_job: Optional[bool] = None, + replica_group_name: Optional[str] = None, ) -> JobModel: if deployment_num is None: deployment_num = run.deployment_num run_spec = RunSpec.parse_raw(run.run_spec) job_spec = ( - await get_job_specs_from_run_spec(run_spec=run_spec, secrets={}, replica_num=replica_num) + await get_job_specs_from_run_spec( + run_spec=run_spec, + secrets={}, + replica_num=replica_num, + replica_group_name=replica_group_name, + ) )[0] job_spec.job_num = job_num job = JobModel( diff --git a/src/dstack/api/server/__init__.py b/src/dstack/api/server/__init__.py index 82b009863a..5888cee80f 100644 --- a/src/dstack/api/server/__init__.py +++ b/src/dstack/api/server/__init__.py @@ -16,6 +16,7 @@ from dstack._internal.utils.logging import get_logger from dstack.api.server._auth import AuthAPIClient from dstack.api.server._backends import BackendsAPIClient +from dstack.api.server._endpoints import EndpointsAPIClient from dstack.api.server._events import EventsAPIClient from dstack.api.server._exports import ExportsAPIClient from dstack.api.server._files import FilesAPIClient @@ -52,6 +53,7 @@ class APIClient: logs: operations with logs gateways: operations with gateways volumes: operations with volumes + endpoints: operations with endpoints exports: operations with exports files: operations with files """ @@ -129,6 +131,10 @@ def gateways(self) -> GatewaysAPIClient: def volumes(self) -> VolumesAPIClient: return VolumesAPIClient(self._request, self._logger) + @property + def endpoints(self) -> EndpointsAPIClient: + return EndpointsAPIClient(self._request, self._logger) + @property def exports(self) -> ExportsAPIClient: return ExportsAPIClient(self._request, self._logger) diff --git a/src/dstack/api/server/_endpoints.py b/src/dstack/api/server/_endpoints.py new file mode 100644 index 0000000000..cb8cb0c4f7 --- /dev/null +++ b/src/dstack/api/server/_endpoints.py @@ -0,0 +1,49 @@ +from typing import List + +from pydantic import parse_obj_as + +from dstack._internal.core.models.endpoints import Endpoint, EndpointConfiguration, EndpointPlan +from dstack._internal.server.schemas.endpoints import ( + CreateEndpointRequest, + DeleteEndpointsRequest, + GetEndpointPlanRequest, + GetEndpointRequest, +) +from dstack.api.server._group import APIClientGroup + + +class EndpointsAPIClient(APIClientGroup): + def list(self, project_name: str) -> List[Endpoint]: + resp = self._request(f"/api/project/{project_name}/endpoints/list") + return parse_obj_as(List[Endpoint.__response__], resp.json()) + + def get(self, project_name: str, name: str) -> Endpoint: + body = GetEndpointRequest(name=name) + resp = self._request(f"/api/project/{project_name}/endpoints/get", body=body.json()) + return parse_obj_as(Endpoint.__response__, resp.json()) + + def get_plan( + self, + project_name: str, + configuration: EndpointConfiguration, + configuration_path: str, + ) -> EndpointPlan: + body = GetEndpointPlanRequest( + configuration=configuration, + configuration_path=configuration_path, + ) + resp = self._request(f"/api/project/{project_name}/endpoints/get_plan", body=body.json()) + return parse_obj_as(EndpointPlan.__response__, resp.json()) + + def create( + self, + project_name: str, + configuration: EndpointConfiguration, + ) -> Endpoint: + body = CreateEndpointRequest(configuration=configuration) + resp = self._request(f"/api/project/{project_name}/endpoints/create", body=body.json()) + return parse_obj_as(Endpoint.__response__, resp.json()) + + def delete(self, project_name: str, names: List[str]) -> None: + body = DeleteEndpointsRequest(names=names) + self._request(f"/api/project/{project_name}/endpoints/delete", body=body.json()) diff --git a/src/tests/_internal/cli/commands/test_logs.py b/src/tests/_internal/cli/commands/test_logs.py new file mode 100644 index 0000000000..53cf05d8a9 --- /dev/null +++ b/src/tests/_internal/cli/commands/test_logs.py @@ -0,0 +1,124 @@ +import argparse +import base64 +from datetime import datetime, timezone +from types import SimpleNamespace +from uuid import uuid4 + +from dstack._internal.cli.commands.logs import LogsCommand +from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.core.models.endpoints import Endpoint, EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.logs import JobSubmissionLogs, LogEvent, LogEventSource + + +def _get_endpoint(run_name: str | None = None) -> Endpoint: + return Endpoint( + id=uuid4(), + name="qwen-endpoint", + project_name="main", + user="test-user", + configuration=EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B"), + created_at=datetime.now(timezone.utc), + last_processed_at=datetime.now(timezone.utc), + status=EndpointStatus.PROVISIONING, + deleted=False, + run_name=run_name, + ) + + +class _FakeRun: + def logs(self, **kwargs): + yield b"service log\n" + + +class _FakeRuns: + def __init__(self, run=None): + self._run = run + self.requested_names = [] + + def get(self, name): + self.requested_names.append(name) + return self._run + + +class _FakeEndpoints: + def __init__(self, endpoint): + self._endpoint = endpoint + + def get(self, project_name, name): + if self._endpoint is None: + raise ResourceNotExistsError() + return self._endpoint + + +class _FakeLogs: + def __init__(self): + self.requests = [] + + def poll(self, project_name, body): + self.requests.append(body) + return JobSubmissionLogs( + logs=[ + LogEvent( + timestamp=datetime.now(timezone.utc), + log_source=LogEventSource.STDOUT, + message=base64.b64encode(b"agent log\n").decode(), + ) + ], + ) + + +def _get_command(endpoint=None, run=None) -> LogsCommand: + command = LogsCommand.__new__(LogsCommand) + command.api = SimpleNamespace( + project="main", + runs=_FakeRuns(run=run), + client=SimpleNamespace( + endpoints=_FakeEndpoints(endpoint=endpoint), + logs=_FakeLogs(), + ), + ) + return command + + +def _get_args(name="qwen-endpoint"): + return argparse.Namespace( + run_name=name, + diagnose=False, + replica=0, + job=0, + ) + + +class TestLogsCommand: + def test_reads_endpoint_agent_logs_when_backing_run_is_not_known(self): + endpoint = _get_endpoint() + command = _get_command(endpoint=endpoint) + + logs = list( + command._get_endpoint_logs(endpoint=endpoint, args=_get_args(), start_time=None) + ) + + assert logs == [b"agent log\n"] + request = command.api.client.logs.requests[0] + assert request.run_name == endpoint.name + assert request.job_submission_id == endpoint.id + + def test_reads_backing_service_logs_when_endpoint_has_run(self): + endpoint = _get_endpoint(run_name="qwen-service") + command = _get_command(endpoint=endpoint, run=_FakeRun()) + + logs = list( + command._get_endpoint_logs(endpoint=endpoint, args=_get_args(), start_time=None) + ) + + assert logs == [b"service log\n"] + assert command.api.runs.requested_names == ["qwen-service"] + + def test_endpoint_name_takes_precedence_over_same_name_run(self): + endpoint = _get_endpoint() + command = _get_command(endpoint=endpoint, run=_FakeRun()) + + logs = list(command._get_logs(args=_get_args(), start_time=None)) + + assert logs == [b"agent log\n"] + assert command.api.runs.requested_names == [] diff --git a/src/tests/_internal/cli/services/configurators/test_endpoint.py b/src/tests/_internal/cli/services/configurators/test_endpoint.py new file mode 100644 index 0000000000..ae40fa6c8b --- /dev/null +++ b/src/tests/_internal/cli/services/configurators/test_endpoint.py @@ -0,0 +1,214 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest +from rich.console import Console + +import dstack._internal.cli.services.configurators.endpoint as endpoint_configurator_module +from dstack._internal.cli.services.configurators.endpoint import ( + _get_apply_confirm_message, + _make_endpoint_url_absolute, + _print_endpoint_plan, + _print_finished_endpoint_message, + _print_submitted_endpoint_message, +) +from dstack._internal.core.models.common import ApplyAction +from dstack._internal.core.models.endpoints import ( + Endpoint, + EndpointConfiguration, + EndpointPlan, + EndpointPresetPolicy, + EndpointProvisioningPlanAgent, + EndpointProvisioningPlanNone, + EndpointProvisioningPlanPreset, + EndpointStatus, +) + + +def _get_endpoint_plan( + provisioning_plan: ( + EndpointProvisioningPlanNone + | EndpointProvisioningPlanPreset + | EndpointProvisioningPlanAgent + ), + current_resource: Endpoint | None = None, + preset_policy: EndpointPresetPolicy = EndpointPresetPolicy.REUSE_OR_CREATE, +) -> EndpointPlan: + configuration = EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B") + return EndpointPlan( + project_name="main", + user="test-user", + configuration=configuration, + configuration_path="endpoint.dstack.yml", + current_resource=current_resource, + action=ApplyAction.CREATE if current_resource is None else ApplyAction.UPDATE, + preset_policy=preset_policy, + provisioning_plan=provisioning_plan, + ) + + +def _get_no_provisioning_plan() -> EndpointProvisioningPlanNone: + return EndpointProvisioningPlanNone( + reason=( + "No matching endpoint presets found. " + "Creating a preset requires the server agent, but " + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + ) + ) + + +def _get_current_endpoint() -> Endpoint: + configuration = EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B") + return Endpoint( + id=uuid4(), + name="qwen-endpoint", + project_name="main", + user="test-user", + configuration=configuration, + created_at=datetime.now(timezone.utc), + last_processed_at=datetime.now(timezone.utc), + status=EndpointStatus.RUNNING, + deleted=False, + ) + + +def _get_failed_endpoint() -> Endpoint: + endpoint = _get_current_endpoint() + endpoint.status = EndpointStatus.FAILED + endpoint.status_message = ( + "Preset policy create requires the server agent, but " + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + ) + return endpoint + + +def _patch_console(monkeypatch: pytest.MonkeyPatch) -> Console: + console = Console(record=True, force_terminal=False, color_system=None, width=120) + monkeypatch.setattr(endpoint_configurator_module, "console", console) + return console + + +class TestPrintEndpointPlan: + def test_prints_stable_properties_for_missing_provisioning_path( + self, monkeypatch: pytest.MonkeyPatch + ): + console = _patch_console(monkeypatch) + plan = _get_endpoint_plan(_get_no_provisioning_plan()) + + _print_endpoint_plan(plan) + + output = console.export_text() + assert "Project main" in output + assert "Endpoint qwen-endpoint" in output + assert "Resources -" in output + assert "Spot policy auto" in output + assert "Max price off" in output + assert "Preset policy reuse-or-create" in output + assert "No matching endpoint presets found" in output + assert "Creating a preset requires the server agent" in output + assert "Model" not in output + assert "Action" not in output + assert "Provisioning" not in output + + def test_prints_agent_reason_when_preset_has_no_offers(self, monkeypatch: pytest.MonkeyPatch): + console = _patch_console(monkeypatch) + plan = _get_endpoint_plan( + EndpointProvisioningPlanAgent( + agent_model="test-agent", + reason="Endpoint preset qwen matched but has no available offers.", + ) + ) + + _print_endpoint_plan(plan) + + output = console.export_text() + assert "Preset policy reuse-or-create" in output + assert "Endpoint preset qwen matched but has no available offers." in output + assert "Agent" not in output + + def test_prints_agent_budget_when_set(self, monkeypatch: pytest.MonkeyPatch): + console = _patch_console(monkeypatch) + plan = _get_endpoint_plan( + EndpointProvisioningPlanAgent(agent_model="test-agent", max_budget=2.0) + ) + + _print_endpoint_plan(plan) + + output = console.export_text() + assert "Agent budget" in output + assert "$2" in output + assert "test-agent" not in output + + +class TestPrintSubmittedEndpointMessage: + def test_prints_generic_detach_message(self, monkeypatch: pytest.MonkeyPatch): + console = _patch_console(monkeypatch) + + _print_submitted_endpoint_message(_get_current_endpoint()) + + output = console.export_text() + assert "Endpoint qwen-endpoint submitted, detaching..." in output + assert "without a provisioning path" not in output + + +class TestPrintFinishedEndpointMessage: + def test_prints_running_url(self, monkeypatch: pytest.MonkeyPatch): + console = _patch_console(monkeypatch) + endpoint = _get_current_endpoint() + endpoint.url = "/proxy/services/main/qwen-endpoint/v1" + + _print_finished_endpoint_message(endpoint) + + output = console.export_text() + assert "/proxy/services/main/qwen-endpoint/v1" in output + + def test_makes_relative_url_absolute(self): + endpoint = _get_current_endpoint() + endpoint.url = "/proxy/services/main/qwen-endpoint/v1" + + _make_endpoint_url_absolute(endpoint, "http://127.0.0.1:8000") + + assert endpoint.url == "http://127.0.0.1:8000/proxy/services/main/qwen-endpoint/v1" + + def test_prints_failed_message_without_logs_hint(self, monkeypatch: pytest.MonkeyPatch): + console = _patch_console(monkeypatch) + + _print_finished_endpoint_message(_get_failed_endpoint()) + + output = console.export_text() + assert "DSTACK_AGENT_ANTHROPIC_API_KEY" in output + assert "dstack logs" not in output + + +class TestGetApplyConfirmMessage: + def test_asks_to_create_without_provisioning_path(self): + plan = _get_endpoint_plan(_get_no_provisioning_plan()) + + assert _get_apply_confirm_message(plan) == "Create the endpoint?" + + def test_asks_to_override_without_provisioning_path(self): + plan = _get_endpoint_plan( + _get_no_provisioning_plan(), + current_resource=_get_current_endpoint(), + ) + + assert ( + _get_apply_confirm_message(plan) + == "Stop and override the endpoint [code]qwen-endpoint[/]?" + ) + + def test_asks_to_create_when_existing_endpoint_is_terminal(self): + plan = _get_endpoint_plan( + _get_no_provisioning_plan(), + current_resource=_get_failed_endpoint(), + ) + + assert _get_apply_confirm_message(plan) == "Create the endpoint?" + + def test_asks_to_override_same_name_run(self): + plan = _get_endpoint_plan(EndpointProvisioningPlanAgent(agent_model="test-agent")) + + assert ( + _get_apply_confirm_message(plan, stop_run_name="qwen-endpoint") + == "Stop and override the run [code]qwen-endpoint[/]?" + ) diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py new file mode 100644 index 0000000000..260c04b1b8 --- /dev/null +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -0,0 +1,171 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +from dstack._internal.cli.utils.endpoint import filter_endpoints_for_listing, get_endpoints_table +from dstack._internal.core.models.endpoints import ( + Endpoint, + EndpointConfiguration, + EndpointStatus, +) + + +def _get_endpoint( + name: str = "qwen-endpoint", + status: EndpointStatus = EndpointStatus.FAILED, + created_at: datetime | None = None, +) -> Endpoint: + if created_at is None: + created_at = datetime.now(timezone.utc) + return Endpoint( + id=uuid4(), + name=name, + project_name="main", + user="test-user", + configuration=EndpointConfiguration(name=name, model="Qwen/Qwen3-0.6B"), + created_at=created_at, + last_processed_at=created_at, + status=status, + status_message="No matching endpoint presets found.", + deleted=False, + ) + + +class TestGetEndpointsTable: + def test_default_table_does_not_show_status_message(self): + table = get_endpoints_table([_get_endpoint()]) + + assert "ERROR" not in [column.header for column in table.columns] + + def test_verbose_table_shows_status_message(self): + table = get_endpoints_table([_get_endpoint()], verbose=True) + + assert "ERROR" in [column.header for column in table.columns] + + def test_status_is_colored(self): + table = get_endpoints_table([_get_endpoint()]) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[indian_red1]failed[/]"] + + def test_running_status_is_colored(self): + table = get_endpoints_table([_get_endpoint(status=EndpointStatus.RUNNING)]) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[bold sea_green3]running[/]"] + + def test_agenting_status_is_colored(self): + table = get_endpoints_table([_get_endpoint(status=EndpointStatus.AGENTING)]) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[bold medium_purple1]agenting[/]"] + + +class TestFilterEndpointsForListing: + def test_default_shows_unfinished_and_latest_finished(self): + endpoints = [ + _get_endpoint( + name="failed-old", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + _get_endpoint( + name="running", + status=EndpointStatus.RUNNING, + created_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + ), + _get_endpoint( + name="failed-new", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), + ), + _get_endpoint( + name="agenting", + status=EndpointStatus.AGENTING, + created_at=datetime(2026, 1, 4, tzinfo=timezone.utc), + ), + ] + + filtered = filter_endpoints_for_listing(endpoints) + + assert [endpoint.name for endpoint in filtered] == [ + "agenting", + "failed-new", + "running", + ] + + def test_default_shows_latest_finished_when_none_are_unfinished(self): + endpoints = [ + _get_endpoint( + name="failed-old", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + _get_endpoint( + name="failed-new", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + ), + ] + + filtered = filter_endpoints_for_listing(endpoints) + + assert [endpoint.name for endpoint in filtered] == ["failed-new"] + + def test_watch_default_shows_only_unfinished(self): + endpoints = [ + _get_endpoint( + name="failed-new", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), + ), + _get_endpoint( + name="agenting", + status=EndpointStatus.AGENTING, + created_at=datetime(2026, 1, 4, tzinfo=timezone.utc), + ), + ] + + filtered = filter_endpoints_for_listing(endpoints, include_latest_finished=False) + + assert [endpoint.name for endpoint in filtered] == ["agenting"] + + def test_all_shows_all_sorted_newest_first(self): + endpoints = [ + _get_endpoint( + name="older", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + _get_endpoint( + name="newer", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + ), + ] + + filtered = filter_endpoints_for_listing(endpoints, show_all=True) + + assert [endpoint.name for endpoint in filtered] == ["newer", "older"] + + def test_last_implies_all(self): + endpoints = [ + _get_endpoint( + name="oldest", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + _get_endpoint( + name="middle", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 2, tzinfo=timezone.utc), + ), + _get_endpoint( + name="newest", + status=EndpointStatus.FAILED, + created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), + ), + ] + + filtered = filter_endpoints_for_listing(endpoints, limit=2) + + assert [endpoint.name for endpoint in filtered] == ["newest", "middle"] diff --git a/src/tests/_internal/core/models/test_endpoints.py b/src/tests/_internal/core/models/test_endpoints.py new file mode 100644 index 0000000000..1656a81716 --- /dev/null +++ b/src/tests/_internal/core/models/test_endpoints.py @@ -0,0 +1,62 @@ +import pytest + +from dstack._internal.core.errors import ConfigurationError +from dstack._internal.core.models.configurations import parse_apply_configuration +from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointPresetPolicy +from dstack._internal.core.models.profiles import CreationPolicy + + +class TestEndpointConfiguration: + def test_parses_endpoint_configuration(self): + conf = parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "env": {"HF_TOKEN": "secret"}, + "fleets": ["gpu-fleet"], + "creation_policy": "reuse", + "preset_policy": "reuse", + "max_agent_budget": 2.0, + } + ) + + assert isinstance(conf, EndpointConfiguration) + assert conf.name == "qwen-endpoint" + assert conf.model == "Qwen/Qwen3-0.6B" + assert conf.env.as_dict() == {"HF_TOKEN": "secret"} + assert conf.fleets is not None + fleet = conf.fleets[0] + assert not isinstance(fleet, str) + assert fleet.name == "gpu-fleet" + assert conf.creation_policy == CreationPolicy.REUSE + assert conf.preset_policy == EndpointPresetPolicy.REUSE + assert conf.max_agent_budget == 2.0 + + def test_defaults_to_reuse_or_create_preset_policy(self): + conf = EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B") + + assert conf.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE + assert conf.max_agent_budget is None + + def test_rejects_non_positive_agent_budget(self): + with pytest.raises(ConfigurationError): + parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "max_agent_budget": 0, + } + ) + + def test_rejects_unknown_fields(self): + with pytest.raises(ConfigurationError): + parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "commands": ["echo not supported"], + } + ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py new file mode 100644 index 0000000000..e73c7f008e --- /dev/null +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -0,0 +1,1386 @@ +import uuid +from datetime import datetime, timezone +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload + +from dstack._internal.core.errors import ServerClientError +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.envs import Env +from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus, ServiceSpec +from dstack._internal.server.background.pipeline_tasks.endpoints import ( + EndpointPipelineItem, + EndpointWorker, +) +from dstack._internal.server.db import get_session_ctx +from dstack._internal.server.models import EndpointModel, EndpointRunSubmissionModel, RunModel +from dstack._internal.server.services.endpoints import record_endpoint_run_submission +from dstack._internal.server.services.endpoints.agent import AgentPlan, AgentProvisioningResult +from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport +from dstack._internal.server.services.endpoints.names import get_endpoint_serving_run_name +from dstack._internal.server.services.endpoints.planning import EndpointPresetPlanningResult +from dstack._internal.server.testing.common import ( + create_job, + create_project, + create_repo, + create_run, + create_user, + list_events, +) + + +@pytest.fixture +def worker() -> EndpointWorker: + return EndpointWorker(queue=Mock(), heartbeater=Mock(), pipeline_hinter=Mock()) + + +class _FakeAgentService: + def __init__(self, result: AgentProvisioningResult | None = None, side_effect=None): + self.provision_endpoint = AsyncMock( + return_value=result or AgentProvisioningResult(), + side_effect=side_effect, + ) + + def is_enabled(self) -> bool: + return True + + def get_plan(self) -> AgentPlan: + return AgentPlan(model="test-agent") + + +def _endpoint_to_pipeline_item(endpoint_model: EndpointModel) -> EndpointPipelineItem: + assert endpoint_model.lock_token is not None + assert endpoint_model.lock_expires_at is not None + return EndpointPipelineItem( + __tablename__=endpoint_model.__tablename__, + id=endpoint_model.id, + lock_token=endpoint_model.lock_token, + lock_expires_at=endpoint_model.lock_expires_at, + prev_lock_expired=False, + status=endpoint_model.status, + to_be_deleted=endpoint_model.to_be_deleted, + ) + + +async def _lock_endpoint_model(session: AsyncSession, endpoint_model: EndpointModel) -> None: + endpoint_model.lock_token = uuid.uuid4() + endpoint_model.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) + await session.commit() + + +async def _create_endpoint_model( + session: AsyncSession, + status: EndpointStatus = EndpointStatus.SUBMITTED, + to_be_deleted: bool = False, + user_ssh_public_key: str | None = None, +) -> EndpointModel: + project = await create_project(session=session) + user = await create_user(session=session, ssh_public_key=user_ssh_public_key) + configuration = EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + env=Env.parse_obj({"HF_TOKEN": "secret"}), + ) + endpoint_model = EndpointModel( + project=project, + user=user, + name=configuration.name, + status=status, + configuration=configuration.json(), + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + last_processed_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + to_be_deleted=to_be_deleted, + ) + endpoint_model.lock_token = uuid.uuid4() + endpoint_model.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) + session.add(endpoint_model) + await session.commit() + return endpoint_model + + +async def _create_backing_service_run( + session: AsyncSession, + endpoint_model: EndpointModel, + status: RunStatus = RunStatus.RUNNING, + job_status: JobStatus = JobStatus.RUNNING, + registered: bool = True, + deleted: bool = False, + service_model_base_url: str | None = "/proxy/services/main/qwen-endpoint-serving/v1", + link_endpoint: bool = True, + run_name: str | None = None, +): + if run_name is None: + run_name = get_endpoint_serving_run_name(endpoint_model.name) + assert run_name is not None + repo = await create_repo( + session=session, + project_id=endpoint_model.project_id, + repo_name=f"{run_name}-repo", + ) + run_spec = RunSpec( + run_name=run_name, + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": run_name, + "commands": [ + "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-0.6B", + "resources": {"gpu": "16GB"}, + } + ), + ) + run = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=endpoint_model.user, + run_name=run_name, + status=status, + run_spec=run_spec, + deleted=deleted, + ) + if service_model_base_url is not None: + run.service_spec = ServiceSpec.parse_obj( + { + "url": "/proxy/services/main/qwen-endpoint-serving/", + "model": { + "name": "Qwen/Qwen3-0.6B", + "base_url": service_model_base_url, + "type": "chat", + }, + } + ).json() + if link_endpoint: + endpoint_model.service_run_id = run.id + await create_job( + session=session, + run=run, + status=job_status, + registered=registered, + ) + await session.commit() + return run + + +async def _create_ready_backing_service_run_for_agent( + endpoint_id: uuid.UUID, +) -> AgentProvisioningResult: + async with get_session_ctx() as session: + res = await session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + endpoint_model = res.unique().scalar_one() + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=_get_agent_run_name(), + ) + return _get_verified_agent_result(run) + + +def _get_verified_agent_result( + run: RunModel, + *, + run_id: uuid.UUID | None = None, + run_name: str | None = None, +) -> AgentProvisioningResult: + return AgentProvisioningResult( + run_id=run_id if run_id is not None else run.id, + run_name=run_name if run_name is not None else run.run_name, + final_report=AgentFinalReport( + success=True, + run_id=run.id, + run_name=run.run_name, + service_yaml=f"type: service\nname: {run.run_name}\n", + verification_summary="Agent verified the model endpoint.", + ), + ) + + +def _get_agent_run_name(suffix: str = "candidate") -> str: + return f"agent-{suffix}" + + +async def _get_endpoint_run_submissions( + session: AsyncSession, + endpoint_model: EndpointModel, +) -> list[EndpointRunSubmissionModel]: + res = await session.execute( + select(EndpointRunSubmissionModel) + .where(EndpointRunSubmissionModel.endpoint_id == endpoint_model.id) + .order_by(EndpointRunSubmissionModel.submission_num) + ) + return list(res.scalars().all()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestRecordEndpointRunSubmission: + async def test_records_ordered_submissions_and_is_idempotent( + self, test_db, session: AsyncSession + ): + endpoint_model = await _create_endpoint_model(session=session) + repo = await create_repo( + session=session, + project_id=endpoint_model.project_id, + repo_name="submissions-repo", + ) + run_1 = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=endpoint_model.user, + run_name="qwen-endpoint-1", + ) + run_2 = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=endpoint_model.user, + run_name="qwen-endpoint-2", + ) + + submission_1 = await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run_1.id, + ) + duplicate_submission_1 = await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run_1.id, + ) + submission_2 = await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run_2.id, + ) + await session.commit() + + assert duplicate_submission_1 is submission_1 + assert submission_1.submission_num == 1 + assert submission_2.submission_num == 2 + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert [submission.run_id for submission in submissions] == [run_1.id, run_2.id] + assert [submission.submission_num for submission in submissions] == [1, 2] + + async def test_rejects_run_already_recorded_for_another_endpoint( + self, test_db, session: AsyncSession + ): + endpoint_model = await _create_endpoint_model(session=session) + other_configuration = EndpointConfiguration( + name="other-qwen-endpoint", + model="Qwen/Qwen3-0.6B", + env=Env.parse_obj({"HF_TOKEN": "secret"}), + ) + other_endpoint_model = EndpointModel( + project=endpoint_model.project, + user=endpoint_model.user, + name=other_configuration.name, + status=EndpointStatus.SUBMITTED, + configuration=other_configuration.json(), + created_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + last_processed_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + ) + session.add(other_endpoint_model) + await session.commit() + repo = await create_repo( + session=session, + project_id=endpoint_model.project_id, + repo_name="recorded-run-repo", + ) + run = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=endpoint_model.user, + run_name="recorded-run", + ) + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run.id, + ) + await session.commit() + + with pytest.raises(ServerClientError, match="Run is already recorded"): + await record_endpoint_run_submission( + session=session, + endpoint_id=other_endpoint_model.id, + run_id=run.id, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestEndpointWorkerSubmitted: + async def test_fails_when_same_name_run_exists_without_link( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.PROVISIONING, + link_endpoint=False, + ) + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(), + ) as find_preset_planning_result_mock, + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", + new=AsyncMock(), + ) as apply_plan_mock, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "Run name 'qwen-endpoint-serving' is taken by an existing run" + ) + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.PROVISIONING + assert run.deleted is False + find_preset_planning_result_mock.assert_not_awaited() + apply_plan_mock.assert_not_awaited() + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Endpoint status changed SUBMITTED -> FAILED " + "(Run name 'qwen-endpoint-serving' is taken by an existing run)" + ) + + async def test_fails_when_run_name_is_taken_by_another_user( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + repo = await create_repo(session=session, project_id=endpoint_model.project_id) + another_user = await create_user(session=session, name="another-user") + run = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=another_user, + run_name=get_endpoint_serving_run_name(endpoint_model.name), + status=RunStatus.RUNNING, + ) + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(), + ) as find_preset_planning_result_mock, + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", + new=AsyncMock(), + ) as apply_plan_mock, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "Run name 'qwen-endpoint-serving' is taken by an existing run" + ) + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.RUNNING + assert run.deleted is False + find_preset_planning_result_mock.assert_not_awaited() + apply_plan_mock.assert_not_awaited() + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Endpoint status changed SUBMITTED -> FAILED " + "(Run name 'qwen-endpoint-serving' is taken by an existing run)" + ) + + async def test_fails_when_run_name_is_taken_by_pre_existing_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + repo = await create_repo(session=session, project_id=endpoint_model.project_id) + run = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=endpoint_model.user, + run_name=get_endpoint_serving_run_name(endpoint_model.name), + status=RunStatus.RUNNING, + submitted_at=datetime(2023, 1, 2, 3, 3, tzinfo=timezone.utc), + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(), + ) as find_preset_planning_result_mock: + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "Run name 'qwen-endpoint-serving' is taken by an existing run" + ) + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.RUNNING + assert run.deleted is False + find_preset_planning_result_mock.assert_not_awaited() + + async def test_finished_same_name_run_does_not_block_preset_submission( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + user_ssh_public_key="ssh-rsa test", + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.DONE, + link_endpoint=False, + ) + preset_plan = Mock() + preset_plan.preset.name = "qwen" + preset_plan.run_plan.run_spec = RunSpec( + run_name="qwen-endpoint-serving", + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint-serving", + "commands": [ + "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-0.6B", + "resources": {"gpu": "16GB"}, + } + ), + ) + preset_plan.run_plan.current_resource = None + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=preset_plan) + ), + ) as find_preset_planning_result_mock, + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", + new=AsyncMock(return_value=Mock(id=run.id)), + ) as apply_plan_mock, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status_message is None + assert endpoint_model.provisioning_method == "preset:qwen" + assert run.status == RunStatus.DONE + assert run.deleted is False + find_preset_planning_result_mock.assert_awaited_once() + apply_plan_mock.assert_awaited_once() + + async def test_submitted_to_failed_without_provisioning_path( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert ( + endpoint_model.status_message == "No matching endpoint presets found. " + "Creating a preset requires the server agent, but " + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + ) + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Endpoint status changed SUBMITTED -> FAILED " + "(No matching endpoint presets found. " + "Creating a preset requires the server agent, but " + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set.)" + ) + + async def test_submitted_to_agenting_with_agent( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + agent_service = _FakeAgentService() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.AGENTING + assert endpoint_model.status_message is None + assert endpoint_model.service_run_id is None + assert endpoint_model.provisioning_method == "agent" + agent_service.provision_endpoint.assert_not_awaited() + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint status changed SUBMITTED -> AGENTING" + + async def test_submitted_to_provisioning_with_matching_preset( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + user_ssh_public_key="ssh-rsa test", + ) + repo = await create_repo(session=session, project_id=endpoint_model.project_id) + run = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=endpoint_model.user, + run_name="qwen-endpoint-submitted", + ) + preset_plan = Mock() + preset_plan.preset.name = "qwen" + preset_plan.run_plan.run_spec = RunSpec( + run_name="qwen-endpoint-serving", + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint-serving", + "commands": [ + "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-0.6B", + "resources": {"gpu": "16GB"}, + } + ), + ) + preset_plan.run_plan.current_resource = None + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=preset_plan) + ), + ) as find_preset_planning_result_mock, + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", + new=AsyncMock(return_value=Mock(id=run.id)), + ) as apply_plan_mock, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status_message is None + assert endpoint_model.service_run_id == run.id + assert endpoint_model.provisioning_method == "preset:qwen" + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert len(submissions) == 1 + assert submissions[0].run_id == run.id + assert submissions[0].submission_num == 1 + find_preset_planning_result_mock.assert_awaited_once() + apply_plan_mock.assert_awaited_once() + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint status changed SUBMITTED -> PROVISIONING" + + async def test_submitted_to_failed_when_preset_submission_fails( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + user_ssh_public_key="ssh-rsa test", + ) + preset_plan = Mock() + preset_plan.preset.name = "qwen" + preset_plan.run_plan.run_spec = RunSpec( + run_name="qwen-endpoint-serving", + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint-serving", + "commands": [ + "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-0.6B", + "resources": {"gpu": "16GB"}, + } + ), + ) + preset_plan.run_plan.current_resource = None + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=preset_plan) + ), + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", + new=AsyncMock(side_effect=ServerClientError("Cannot override active run")), + ) as apply_plan_mock, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "Cannot override active run" + apply_plan_mock.assert_awaited_once() + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message + == "Endpoint status changed SUBMITTED -> FAILED (Cannot override active run)" + ) + + async def test_delete_request_after_fetch_prevents_preset_submission( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + user_ssh_public_key="ssh-rsa test", + ) + item = _endpoint_to_pipeline_item(endpoint_model) + endpoint_model.to_be_deleted = True + await session.commit() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(), + ) as find_preset_planning_result_mock, + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", + new=AsyncMock(), + ) as apply_plan_mock, + ): + await worker.process(item) + + await session.refresh(endpoint_model) + assert endpoint_model.deleted is True + assert endpoint_model.deleted_at is not None + find_preset_planning_result_mock.assert_not_awaited() + apply_plan_mock.assert_not_awaited() + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint deleted" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestEndpointWorkerProvisioning: + async def test_agenting_does_not_fail_on_legacy_same_name_run_conflict( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + legacy_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.PROVISIONING, + link_endpoint=False, + ) + agent_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=_get_agent_run_name(), + ) + agent_service = _FakeAgentService(result=_get_verified_agent_result(agent_run)) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(legacy_run) + await session.refresh(agent_run) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status_message is None + assert endpoint_model.service_run_id == agent_run.id + assert legacy_run.status == RunStatus.PROVISIONING + assert legacy_run.deleted is False + assert agent_run.deleted is False + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint status changed AGENTING -> PROVISIONING" + + async def test_fails_when_agent_reports_foreign_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + repo = await create_repo(session=session, project_id=endpoint_model.project_id) + another_user = await create_user(session=session, name="another-user") + run = await create_run( + session=session, + project=endpoint_model.project, + repo=repo, + user=another_user, + run_name=_get_agent_run_name(), + status=RunStatus.RUNNING, + ) + agent_service = _FakeAgentService(result=_get_verified_agent_result(run)) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "Run 'agent-candidate' is not owned by the endpoint user" + ) + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.RUNNING + assert run.deleted is False + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Endpoint status changed AGENTING -> FAILED " + "(Run 'agent-candidate' is not owned by the endpoint user)" + ) + + async def test_agent_creates_ready_service_and_endpoint_becomes_running( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + preset_service = Mock() + preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) + + async def provision_endpoint(endpoint_model, pipeline_hinter): + return await _create_ready_backing_service_run_for_agent(endpoint_model.id) + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.AGENTING + await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.service_run_id is not None + await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + assert endpoint_model.status_message is None + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert len(submissions) == 1 + assert submissions[0].run_id == endpoint_model.service_run_id + assert submissions[0].submission_num == 1 + agent_service.provision_endpoint.assert_awaited_once() + preset_service.save_preset.assert_awaited_once() + + async def test_agent_reported_run_id_links_backing_service_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + + async def provision_endpoint(endpoint_model, pipeline_hinter): + async with get_session_ctx() as provisioning_session: + res = await provisioning_session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_model.id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + provisioning_endpoint = res.unique().scalar_one() + run = await _create_backing_service_run( + session=provisioning_session, + endpoint_model=provisioning_endpoint, + link_endpoint=False, + run_name=_get_agent_run_name(), + ) + return AgentProvisioningResult( + run_id=run.id, + final_report=AgentFinalReport( + success=True, + run_id=run.id, + run_name=run.run_name, + service_yaml=f"type: service\nname: {run.run_name}\n", + verification_summary="Agent verified the model endpoint.", + ), + ) + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.service_run_id is not None + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert len(submissions) == 1 + assert submissions[0].run_id == endpoint_model.service_run_id + + async def test_agent_reported_run_without_verification_report_fails_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + + async def provision_endpoint(endpoint_model, pipeline_hinter): + async with get_session_ctx() as provisioning_session: + res = await provisioning_session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_model.id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + provisioning_endpoint = res.unique().scalar_one() + run = await _create_backing_service_run( + session=provisioning_session, + endpoint_model=provisioning_endpoint, + link_endpoint=False, + ) + return AgentProvisioningResult(run_id=run.id) + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.service_run_id is None + assert endpoint_model.status_message == "Server agent did not return a verification report" + + async def test_agent_failure_fails_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + agent_service = _FakeAgentService( + result=AgentProvisioningResult(error="agent could not find a deployable recipe") + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "agent could not find a deployable recipe" + agent_service.provision_endpoint.assert_awaited_once() + + async def test_agent_failure_does_not_stop_unlinked_same_name_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + created_run_id = None + + async def provision_endpoint(endpoint_model, pipeline_hinter): + nonlocal created_run_id + async with get_session_ctx() as agent_session: + res = await agent_session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_model.id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + endpoint = res.unique().scalar_one() + run = await _create_backing_service_run( + session=agent_session, + endpoint_model=endpoint, + link_endpoint=False, + ) + created_run_id = run.id + return AgentProvisioningResult(error="agent could not verify the service") + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + assert created_run_id is not None + await session.refresh(endpoint_model) + run = await session.get(RunModel, created_run_id) + assert run is not None + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "agent could not verify the service" + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.RUNNING + agent_service.provision_endpoint.assert_awaited_once() + + async def test_waits_when_backing_run_is_not_ready( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.PROVISIONING, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status_message is None + events = await list_events(session) + assert events == [] + + async def test_moves_to_running_when_backing_service_is_ready( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + await _create_backing_service_run(session=session, endpoint_model=endpoint_model) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + assert endpoint_model.status_message is None + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint status changed PROVISIONING -> RUNNING" + + async def test_saves_preset_when_agent_backing_service_becomes_running( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + endpoint_model.provisioning_method = "agent" + await _create_backing_service_run(session=session, endpoint_model=endpoint_model) + preset_service = Mock() + preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + preset_service.save_preset.assert_awaited_once() + saved_preset = preset_service.save_preset.await_args.args[0] + assert saved_preset.model == "Qwen/Qwen3-0.6B" + assert [group.name for group in saved_preset.replica_spec_groups] == ["0"] + comments = preset_service.save_preset.await_args.kwargs["comments"] + assert f"endpoint: {endpoint_model.name}" in comments + + async def test_preset_save_failure_does_not_block_agent_endpoint_activation( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + endpoint_model.provisioning_method = "agent" + await _create_backing_service_run(session=session, endpoint_model=endpoint_model) + preset_service = Mock() + preset_service.save_preset = AsyncMock(side_effect=OSError("read-only preset dir")) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + preset_service.save_preset.assert_awaited_once() + + async def test_waits_when_backing_service_has_no_registered_jobs( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + registered=False, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status_message is None + events = await list_events(session) + assert events == [] + + async def test_fails_when_backing_run_finished( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.FAILED, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "Backing service run finished with status failed" + assert run.status == RunStatus.FAILED + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Endpoint status changed PROVISIONING -> FAILED " + "(Backing service run finished with status failed)" + ) + + async def test_stops_running_backing_run_when_endpoint_fails( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROVISIONING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + ) + run.service_spec = "invalid" + await session.commit() + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "Backing service spec is invalid" + assert run.status == RunStatus.TERMINATING + events = await list_events(session) + messages = [event.message for event in events] + assert any( + message.startswith("Run status changed RUNNING -> TERMINATING") for message in messages + ) + assert ( + "Endpoint status changed PROVISIONING -> FAILED " + "(Backing service spec is invalid)" in messages + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestEndpointWorkerActive: + async def test_ready_backing_service_keeps_endpoint_active( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + ) + endpoint_model.status_message = "previous failure" + await _create_backing_service_run(session=session, endpoint_model=endpoint_model) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + assert endpoint_model.status_message is None + events = await list_events(session) + assert events == [] + + async def test_not_ready_backing_service_keeps_endpoint_active( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + ) + await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + registered=False, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + assert endpoint_model.status_message is None + events = await list_events(session) + assert events == [] + + async def test_finished_backing_run_fails_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.FAILED, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "Backing service run finished with status failed" + assert run.status == RunStatus.FAILED + events = await list_events(session) + assert len(events) == 1 + assert ( + events[0].message == "Endpoint status changed RUNNING -> FAILED " + "(Backing service run finished with status failed)" + ) + + async def test_stops_running_backing_run_when_endpoint_fails( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + ) + run.service_spec = "invalid" + await session.commit() + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "Backing service spec is invalid" + assert run.status == RunStatus.TERMINATING + events = await list_events(session) + messages = [event.message for event in events] + assert any( + message.startswith("Run status changed RUNNING -> TERMINATING") for message in messages + ) + assert ( + "Endpoint status changed RUNNING -> FAILED (Backing service spec is invalid)" + in messages + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestEndpointWorkerDeleted: + async def test_marks_endpoint_deleted( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.FAILED, + to_be_deleted=True, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.deleted is True + assert endpoint_model.deleted_at is not None + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint deleted" + + async def test_stops_active_backing_run_before_deleting_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + to_be_deleted=True, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.RUNNING, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.deleted is False + assert endpoint_model.deleted_at is None + assert run.status == RunStatus.TERMINATING + assert run.deleted is False + + async def test_deletes_endpoint_without_stopping_unlinked_same_name_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + to_be_deleted=True, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.RUNNING, + link_endpoint=False, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.deleted is True + assert endpoint_model.deleted_at is not None + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.RUNNING + assert run.deleted is False + + async def test_deletes_finished_backing_run_before_deleting_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.RUNNING, + to_be_deleted=True, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.TERMINATED, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.deleted is True + assert endpoint_model.deleted_at is not None + assert run.deleted is True + events = await list_events(session) + assert [event.message for event in events] == [ + "Run deleted", + "Endpoint deleted", + ] diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py new file mode 100644 index 0000000000..6d85360f89 --- /dev/null +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -0,0 +1,521 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, Mock, patch +from uuid import UUID + +import pytest +from freezegun import freeze_time +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.instances import ( + InstanceAvailability, + InstanceOfferWithAvailability, +) +from dstack._internal.core.models.runs import RunSpec +from dstack._internal.core.models.users import GlobalRole, ProjectRole +from dstack._internal.server import settings +from dstack._internal.server.models import EndpointModel +from dstack._internal.server.services.endpoints.agent import AgentPlan +from dstack._internal.server.services.endpoints.planning import ( + EndpointPresetPlan, + EndpointPresetPlanningResult, +) +from dstack._internal.server.services.endpoints.presets import ( + EndpointPreset, + EndpointPresetReplicaSpecGroup, +) +from dstack._internal.server.services.projects import add_project_member +from dstack._internal.server.testing.common import ( + create_project, + create_user, + get_auth_headers, + list_events, +) + + +class TestEndpointPlan: + @pytest.mark.asyncio + async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/get_plan") + assert response.status_code in [401, 403] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_none_provisioning_plan( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + + with patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "env": {"HF_TOKEN": "secret"}, + }, + "configuration_path": "endpoint.dstack.yml", + }, + ) + + assert response.status_code == 200, response.json() + body = response.json() + assert body["project_name"] == project.name + assert body["user"] == user.name + assert body["configuration"]["model"] == "Qwen/Qwen3-0.6B" + assert body["configuration_path"] == "endpoint.dstack.yml" + assert body["current_resource"] is None + assert body["action"] == "create" + assert body["preset_policy"] == "reuse-or-create" + assert body["provisioning_plan"] == { + "type": "none", + "reason": ( + "No matching endpoint presets found. " + "Creating a preset requires the server agent, but " + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + ), + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_create_plan_when_existing_endpoint_is_terminal( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.FAILED, + ) + + with patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + }, + }, + ) + + assert response.status_code == 200, response.json() + body = response.json() + assert body["current_resource"] is None + assert body["action"] == "create" + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_agent_provisioning_plan( + self, test_db, session: AsyncSession, client: AsyncClient, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MAX_BUDGET", 4.0) + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + agent_service = Mock() + agent_service.is_enabled.return_value = True + agent_service.get_plan.return_value = AgentPlan(model="test-agent") + + with ( + patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.services.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "max_agent_budget": 2.0, + }, + "configuration_path": "endpoint.dstack.yml", + }, + ) + + assert response.status_code == 200, response.json() + body = response.json() + assert body["preset_policy"] == "reuse-or-create" + assert body["provisioning_plan"] == { + "type": "agent", + "agent_model": "test-agent", + "max_budget": 2.0, + "reason": None, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_preset_provisioning_plan( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + + with patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=_endpoint_preset_plan()) + ), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + }, + }, + ) + + assert response.status_code == 200, response.json() + body = response.json() + assert body["provisioning_plan"]["type"] == "preset" + assert body["preset_policy"] == "reuse-or-create" + assert body["provisioning_plan"]["preset_name"] == "qwen" + assert body["provisioning_plan"]["service_name"] == "qwen-endpoint-serving" + assert body["provisioning_plan"]["replica_spec_groups"][0]["name"] == "0" + assert body["provisioning_plan"]["job_offers"][0]["replica_group"] == "0" + assert body["provisioning_plan"]["job_offers"][0]["resources"]["gpu"] is not None + assert body["provisioning_plan"]["job_offers"][0]["spot"] is None + assert body["provisioning_plan"]["job_offers"][0]["max_price"] is None + assert body["provisioning_plan"]["job_offers"][0]["offers"][0]["backend"] == "aws" + assert body["provisioning_plan"]["job_offers"][0]["total_offers"] == 2 + + +class TestCreateEndpoint: + @pytest.mark.asyncio + async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/create") + assert response.status_code in [401, 403] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + @freeze_time(datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc)) + async def test_creates_endpoint(self, test_db, session: AsyncSession, client: AsyncClient): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "env": {"HF_TOKEN": "secret"}, + } + }, + ) + + assert response.status_code == 200, response.json() + body = response.json() + UUID(body["id"]) + assert body["name"] == "qwen-endpoint" + assert body["project_name"] == project.name + assert body["user"] == user.name + assert body["configuration"]["model"] == "Qwen/Qwen3-0.6B" + assert body["configuration"]["env"] == {"HF_TOKEN": "secret"} + assert body["created_at"] == "2023-01-02T03:04:00+00:00" + assert body["last_processed_at"] == "2023-01-02T03:04:00+00:00" + assert body["status"] == "submitted" + assert body["status_message"] is None + assert body["deleted"] is False + assert body["deleted_at"] is None + assert body["run_name"] is None + assert body["url"] is None + assert body["error"] is None + + res = await session.execute(select(EndpointModel)) + assert res.scalar_one() + events = await list_events(session) + assert len(events) == 1 + assert events[0].message == "Endpoint created. Status: SUBMITTED" + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_recreates_terminal_endpoint( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + previous_endpoint = await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.FAILED, + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + } + }, + ) + + assert response.status_code == 200, response.json() + new_endpoint_id = UUID(response.json()["id"]) + assert new_endpoint_id != previous_endpoint.id + await session.refresh(previous_endpoint) + assert previous_endpoint.deleted is True + assert previous_endpoint.deleted_at is not None + new_endpoint = await session.get(EndpointModel, new_endpoint_id) + assert new_endpoint is not None + assert new_endpoint.name == previous_endpoint.name + assert new_endpoint.status == EndpointStatus.SUBMITTED + assert new_endpoint.deleted is False + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_rejects_duplicate_non_terminal_endpoint( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.SUBMITTED, + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"][0]["code"] == "resource_exists" + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_rejects_unresolved_env_sentinel( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "env": ["HF_TOKEN"], + } + }, + ) + + assert response.status_code == 400 + assert "Endpoint env is unresolved" in response.json()["detail"][0]["msg"] + + +class TestGetEndpoint: + @pytest.mark.asyncio + async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/get") + assert response.status_code in [401, 403] + + +class TestDeleteEndpoint: + @pytest.mark.asyncio + async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/delete") + assert response.status_code in [401, 403] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_marks_endpoint_for_deletion( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + create_response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "env": {"HF_TOKEN": "secret"}, + } + }, + ) + assert create_response.status_code == 200, create_response.json() + + response = await client.post( + f"/api/project/{project.name}/endpoints/delete", + headers=get_auth_headers(user.token), + json={"names": ["qwen-endpoint"]}, + ) + + assert response.status_code == 200, response.json() + res = await session.execute(select(EndpointModel)) + endpoint_model = res.scalar_one() + assert endpoint_model.to_be_deleted + assert endpoint_model.deleted is False + assert endpoint_model.deleted_at is None + + get_response = await client.post( + f"/api/project/{project.name}/endpoints/get", + headers=get_auth_headers(user.token), + json={"name": "qwen-endpoint"}, + ) + assert get_response.status_code == 200 + + events = await list_events(session) + assert [e.message for e in events] == [ + "Endpoint created. Status: SUBMITTED", + "Endpoint marked for deletion", + ] + + +async def _create_endpoint_model( + session: AsyncSession, + project, + user, + status: EndpointStatus = EndpointStatus.SUBMITTED, + name: str = "qwen-endpoint", +) -> EndpointModel: + configuration = EndpointConfiguration(name=name, model="Qwen/Qwen3-0.6B") + endpoint_model = EndpointModel( + name=name, + project=project, + user=user, + configuration=configuration.json(), + status=status, + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + last_processed_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + session.add(endpoint_model) + await session.commit() + return endpoint_model + + +def _endpoint_preset_plan() -> EndpointPresetPlan: + service_configuration = ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint-serving", + "commands": [ + "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-0.6B", + "resources": {"gpu": "16GB"}, + } + ) + preset = EndpointPreset( + name="qwen", + model="Qwen/Qwen3-0.6B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "0", "replica_specs": [{"gpu": "16GB"}]} + ) + ], + configuration=service_configuration, + ) + run_plan = Mock() + run_plan.get_effective_run_spec.return_value = RunSpec( + run_name="qwen-endpoint-serving", + configuration=service_configuration, + ) + job_plan = Mock() + job_plan.job_spec.replica_group = "0" + job_plan.job_spec.requirements.resources = service_configuration.resources + job_plan.job_spec.requirements.spot = None + job_plan.job_spec.requirements.max_price = None + job_plan.offers = [_instance_offer()] + job_plan.total_offers = 2 + job_plan.max_price = 1.25 + run_plan.job_plans = [job_plan] + return EndpointPresetPlan(preset=preset, run_plan=run_plan) + + +def _instance_offer() -> InstanceOfferWithAvailability: + return InstanceOfferWithAvailability.parse_obj( + { + "backend": BackendType.AWS, + "region": "us-east-1", + "price": 1.25, + "availability": InstanceAvailability.AVAILABLE, + "instance": { + "name": "g5.xlarge", + "resources": { + "cpus": 4, + "memory_mib": 16384, + "gpus": [], + "spot": False, + }, + }, + } + ) diff --git a/src/tests/_internal/server/services/endpoints/test_agent_report.py b/src/tests/_internal/server/services/endpoints/test_agent_report.py new file mode 100644 index 0000000000..367c730d55 --- /dev/null +++ b/src/tests/_internal/server/services/endpoints/test_agent_report.py @@ -0,0 +1,60 @@ +import uuid + +import pytest +from pydantic import ValidationError + +from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport + + +class TestAgentFinalReport: + def test_accepts_success_report(self): + run_id = uuid.uuid4() + + report = AgentFinalReport.parse_obj( + { + "success": True, + "run_id": str(run_id), + "service_yaml": "type: service\nname: qwen-serving\n", + "recipe_sources": ["https://example.com/recipe"], + "verification_summary": "Chat completion request returned 200.", + } + ) + + assert report.run_id == run_id + assert report.recipe_sources == ["https://example.com/recipe"] + + def test_rejects_success_without_verification_summary(self): + with pytest.raises(ValidationError, match="verification_summary"): + AgentFinalReport.parse_obj( + { + "success": True, + "run_id": str(uuid.uuid4()), + "run_name": "qwen-serving", + "service_yaml": "type: service\nname: qwen-serving\n", + } + ) + + def test_rejects_success_without_run_id(self): + with pytest.raises(ValidationError, match="run_id"): + AgentFinalReport.parse_obj( + { + "success": True, + "run_name": "qwen-serving", + "service_yaml": "type: service\nname: qwen-serving\n", + "verification_summary": "Chat completion request returned 200.", + } + ) + + def test_accepts_failure_report(self): + report = AgentFinalReport.parse_obj( + { + "success": False, + "failure_summary": "No deployable recipe matched the budget.", + } + ) + + assert report.failure_summary == "No deployable recipe matched the budget." + + def test_rejects_failure_without_summary(self): + with pytest.raises(ValidationError, match="failure_summary"): + AgentFinalReport.parse_obj({"success": False}) diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py new file mode 100644 index 0000000000..a85d6b61af --- /dev/null +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -0,0 +1,683 @@ +import json +import uuid +from asyncio import StreamReader +from datetime import datetime, timezone + +import pytest +import yaml +from sqlalchemy.ext.asyncio import AsyncSession + +import dstack._internal.server.services.endpoints.agent.claude as claude_module +from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.envs import Env +from dstack._internal.core.models.profiles import SpotPolicy +from dstack._internal.server import settings +from dstack._internal.server.models import EndpointModel +from dstack._internal.server.services.endpoints.agent.claude import ( + ClaudeAgentService, + _AgentRunnerResult, + _AgentWorkspace, + _build_claude_command, + _read_agent_stdout, + _run_agent_in_subprocess, + _write_trace_record, + get_claude_agent_unavailable_reason, +) +from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport +from dstack._internal.server.testing.common import create_project, create_user + + +async def _create_endpoint_model( + session: AsyncSession, + max_agent_budget: float | None = None, + max_price: float | None = None, + spot_policy: SpotPolicy | None = None, +) -> EndpointModel: + user = await create_user(session=session, name="admin", token="user-token") + project = await create_project(session=session, owner=user, name="main") + configuration = EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + env=Env.parse_obj({"HF_TOKEN": "hf-secret"}), + max_agent_budget=max_agent_budget, + max_price=max_price, + spot_policy=spot_policy, + ) + endpoint_model = EndpointModel( + id=uuid.uuid4(), + name=configuration.name, + project=project, + user=user, + status=EndpointStatus.PROVISIONING, + provisioning_method="agent", + configuration=configuration.json(), + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + last_processed_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + session.add(endpoint_model) + await session.commit() + return endpoint_model + + +def _configure_fake_claude(tmp_path, monkeypatch: pytest.MonkeyPatch) -> str: + claude_path = tmp_path / "claude" + claude_path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + claude_path.chmod(0o755) + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", str(claude_path)) + return str(claude_path) + + +class TestClaudeAgentService: + @pytest.mark.asyncio + async def test_invokes_agent_with_isolated_dstack_cli_context( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + claude_path = _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MODEL", "test-claude-model") + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MAX_BUDGET", 3.0) + monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") + monkeypatch.setenv("DATABASE_URL", "must-not-leak") + captured = {} + run_id = uuid.uuid4() + + async def runner(workspace, request): + captured["workspace"] = workspace + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=run_id, + run_name="qwen-agent-candidate", + service_yaml="type: service\nname: qwen-agent-candidate\n", + verification_summary="Agent verified chat completions.", + ) + ) + + endpoint_model = await _create_endpoint_model(session, max_agent_budget=1.5) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + assert result.run_id == run_id + assert result.run_name == "qwen-agent-candidate" + assert result.final_report is not None + assert result.final_report.verification_summary == "Agent verified chat completions." + assert ( + "successful final report must include the verified service run `run_id`" + in captured["request"]["prompt"] + ) + assert captured["request"]["cwd"] == str(tmp_path / str(endpoint_model.id) / "workspace") + env = captured["request"]["env"] + assert env["ANTHROPIC_API_KEY"] == "agent-secret" + assert env["HOME"] == str(tmp_path / str(endpoint_model.id) / "home") + assert env["DSTACK_PROJECT"] == "main" + assert env["HF_TOKEN"] == "hf-secret" + assert "DATABASE_URL" not in env + assert captured["request"]["options"]["model"] == "test-claude-model" + assert captured["request"]["options"]["max_budget"] == 1.5 + assert "StructuredOutput" in captured["request"]["options"]["tools"] + assert "Edit" in captured["request"]["options"]["allowed_tools"] + assert "StructuredOutput" in captured["request"]["options"]["allowed_tools"] + schema_text = json.dumps(captured["request"]["options"]["json_schema"]) + assert '"title"' not in schema_text + assert '"format"' not in schema_text + command = _build_claude_command(captured["request"]) + assert command[0] == claude_path + assert command[command.index("--tools") + 1] == captured["request"]["options"]["tools"] + assert "--permission-mode" in command + assert command[command.index("--permission-mode") + 1] == "bypassPermissions" + assert "--max-budget-usd" in command + assert command[command.index("--max-budget-usd") + 1] == "1.5" + config = yaml.safe_load( + (tmp_path / str(endpoint_model.id) / "home" / ".dstack" / "config.yml").read_text() + ) + assert config == { + "projects": [ + { + "name": "main", + "url": "http://127.0.0.1:8000", + "token": "user-token", + "default": True, + } + ], + } + work_dir = tmp_path / str(endpoint_model.id) / "workspace" + state = json.loads((work_dir / "agent_state.json").read_text()) + assert state["endpoint_name"] == "qwen-endpoint" + assert state["model"] == "Qwen/Qwen3-0.6B" + assert state["phase"] == "success" + assert state["max_agent_budget"] == 1.5 + assert (work_dir / "sources.jsonl").exists() + assert (work_dir / "candidates.jsonl").exists() + assert (work_dir / "commands.jsonl").exists() + assert (work_dir / "hardware_reasoning.md").exists() + final_report = json.loads((work_dir / "final_report.json").read_text()) + assert final_report["success"] is True + assert final_report["run_name"] == "qwen-agent-candidate" + + @pytest.mark.asyncio + async def test_includes_endpoint_profile_constraints_in_prompt( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + captured = {} + + async def runner(workspace, request): + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=uuid.uuid4(), + run_name="qwen-agent-candidate", + service_yaml="type: service\nname: qwen-agent-candidate\n", + verification_summary="Agent verified chat completions.", + ) + ) + + endpoint_model = await _create_endpoint_model( + session, + max_price=0.3, + spot_policy=SpotPolicy.ONDEMAND, + ) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + prompt = captured["request"]["prompt"] + assert "- max_price: 0.3" in prompt + assert "- spot_policy: on-demand" in prompt + assert "--max-price 0.3 --on-demand" in prompt + assert "printf 'n\\n' | dstack apply" in prompt + assert "Do not use `dstack ps` table text for machine decisions" in prompt + assert "stopped_by_user" in prompt + assert "Do not blindly select the cheapest offer" in prompt + assert "--availability-zone" not in prompt + assert "--instance " not in prompt + + @pytest.mark.asyncio + async def test_prompt_includes_real_deployment_harness_context( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + captured = {} + + async def runner(workspace, request): + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=uuid.uuid4(), + run_name="qwen-agent-candidate", + service_yaml="type: service\nname: qwen-agent-candidate\n", + verification_summary="Agent verified chat completions.", + ) + ) + + endpoint_model = await _create_endpoint_model(session) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + prompt = captured["request"]["prompt"] + assert "This is a real deployment investigation" in prompt + assert "Do not call hidden server APIs" in prompt + assert "dstack run get --json" in prompt + assert "https://docs.vllm.ai/" in prompt + assert "https://docs.sglang.ai/" in prompt + assert "https://huggingface.co/Qwen/Qwen3-0.6B" in prompt + assert "https://www.wafer.ai/blog/glm52-amd" in prompt + assert "https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development/" in prompt + assert "Prototypes may be dstack services, tasks, or dev environments" in prompt + assert "final verified run reported to the server must be a dstack service" in prompt + assert "do not attempt P/D disaggregation" in prompt + + @pytest.mark.asyncio + async def test_uses_server_default_agent_budget_when_endpoint_does_not_set_one( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MAX_BUDGET", 4.0) + captured = {} + + async def runner(workspace, request): + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=uuid.uuid4(), + run_name="qwen-agent-candidate", + service_yaml="type: service\nname: qwen-agent-candidate\n", + verification_summary="Agent verified chat completions.", + ) + ) + + endpoint_model = await _create_endpoint_model(session) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + assert captured["request"]["options"]["max_budget"] == 4.0 + + @pytest.mark.asyncio + async def test_writes_redacted_trace_in_debug_mode( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "LOG_LEVEL", "DEBUG") + run_id = uuid.uuid4() + + async def runner(workspace, request): + _write_trace_record( + workspace, + {"type": "FakeMessage", "text": "hf-secret agent-secret"}, + ) + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=run_id, + run_name="qwen-agent-candidate", + service_yaml="token: hf-secret\n", + verification_summary="Agent verified with agent-secret.", + ) + ) + + endpoint_model = await _create_endpoint_model(session) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + trace_path = tmp_path / str(endpoint_model.id) / "trace.jsonl" + trace = trace_path.read_text() + assert "agent-secret" not in trace + assert "hf-secret" not in trace + assert "[redacted]" in trace + event = json.loads(trace.splitlines()[0]) + assert event["type"] == "FakeMessage" + + @pytest.mark.asyncio + async def test_returns_error_when_sdk_fails_before_report( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + + async def runner(workspace, request): + return _AgentRunnerResult( + error="Server agent failed before returning a verification report: bad key" + ) + + endpoint_model = await _create_endpoint_model(session) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert ( + result.error == "Server agent failed before returning a verification report: bad key" + ) + assert result.final_report is None + work_dir = tmp_path / str(endpoint_model.id) / "workspace" + state = json.loads((work_dir / "agent_state.json").read_text()) + assert state["phase"] == "failure" + agent_error = json.loads((work_dir / "agent_error.json").read_text()) + assert agent_error["success"] is False + assert agent_error["failure_summary"] == ( + "Server agent failed before returning a verification report: bad key" + ) + + def test_returns_error_when_configured_claude_path_is_not_executable( + self, + tmp_path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", str(tmp_path / "missing-claude")) + + assert get_claude_agent_unavailable_reason() == ( + "DSTACK_AGENT_CLAUDE_PATH does not resolve to an executable: " + f"{tmp_path / 'missing-claude'}" + ) + + def test_returns_error_when_agent_key_is_blank(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "") + + assert get_claude_agent_unavailable_reason() == ( + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + ) + + def test_falls_back_to_claude_in_path(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", None) + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + monkeypatch.setattr( + claude_module.shutil, + "which", + lambda name: "/usr/local/bin/claude" if name == "claude" else None, + ) + + command = _build_claude_command( + { + "prompt": "prompt", + "options": { + "allowed_tools": "Bash", + "disallowed_tools": "", + "model": "test-model", + "max_turns": 1, + "max_budget": None, + "json_schema": {}, + }, + } + ) + + assert get_claude_agent_unavailable_reason() is None + assert command[0] == "/usr/local/bin/claude" + + @pytest.mark.asyncio + async def test_reads_claude_stream_incrementally(self, tmp_path): + reader = StreamReader() + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=tmp_path / "trace.jsonl", + env={}, + redacted_values=["secret"], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + max_agent_budget=None, + ) + report = { + "success": True, + "run_id": str(uuid.uuid4()), + "run_name": "qwen-agent-candidate", + "service_yaml": "env: secret\n", + "verification_summary": "verified secret", + } + reader.feed_data( + json.dumps({"type": "assistant", "message": {"content": "working"}}).encode() + b"\n" + ) + reader.feed_data( + json.dumps({"type": "result", "result": json.dumps(report)}).encode() + b"\n" + ) + reader.feed_eof() + + output = await _read_agent_stdout(reader, workspace) + + assert output.report_data == report + trace = (tmp_path / "trace.jsonl").read_text() + assert "secret" not in trace + assert "[redacted]" in trace + + @pytest.mark.asyncio + async def test_writes_compact_agent_logs_from_claude_stream(self, tmp_path): + class FakeLogWriter: + def __init__(self): + self.messages = [] + + async def write(self, message): + self.messages.append(message) + + async def flush(self): + pass + + reader = StreamReader() + log_writer = FakeLogWriter() + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=None, + env={}, + redacted_values=["secret"], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + max_agent_budget=None, + log_writer=log_writer, + ) + reader.feed_data( + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "id": "tool-1", + "name": "Bash", + "input": { + "description": "List offers", + "command": "dstack offer --gpu 1 --max-price 0.3", + }, + } + ] + }, + } + ).encode() + + b"\n" + ) + reader.feed_data( + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "tool-1", + "content": "No offers secret", + "is_error": False, + } + ] + }, + } + ).encode() + + b"\n" + ) + reader.feed_eof() + + await _read_agent_stdout(reader, workspace) + + assert log_writer.messages == [ + "Agent tool: Bash (List offers)\n$ dstack offer --gpu 1 --max-price 0.3", + "Tool output:\nNo offers [redacted]", + ] + command_records = [ + json.loads(line) + for line in (tmp_path / "work" / "commands.jsonl").read_text().splitlines() + ] + assert command_records[0]["event"] == "tool_use" + assert command_records[0]["command"] == "dstack offer --gpu 1 --max-price 0.3" + assert command_records[1]["event"] == "tool_result" + assert command_records[1]["tool_use_id"] == "tool-1" + output = (tmp_path / "work" / command_records[1]["output_path"]).read_text() + assert output == "No offers [redacted]" + + @pytest.mark.asyncio + async def test_truncates_large_tool_outputs_in_endpoint_logs(self, tmp_path): + class FakeLogWriter: + def __init__(self): + self.messages = [] + + async def write(self, message): + self.messages.append(message) + + async def flush(self): + pass + + reader = StreamReader() + log_writer = FakeLogWriter() + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=None, + env={}, + redacted_values=[], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + max_agent_budget=None, + log_writer=log_writer, + ) + long_output = "\n".join(f"offer {i:04d} gpu=A5000 price=0.27" for i in range(200)) + reader.feed_data( + json.dumps( + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "tool-1", + "content": long_output, + "is_error": False, + } + ] + }, + } + ).encode() + + b"\n" + ) + reader.feed_eof() + + await _read_agent_stdout(reader, workspace) + + assert len(log_writer.messages) == 1 + assert "Full output is stored in the endpoint agent workspace" in log_writer.messages[0] + assert "offer 0199" not in log_writer.messages[0] + command_records = [ + json.loads(line) + for line in (tmp_path / "work" / "commands.jsonl").read_text().splitlines() + ] + output = (tmp_path / "work" / command_records[0]["output_path"]).read_text() + assert output == long_output + + @pytest.mark.asyncio + async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missing( + self, + tmp_path, + monkeypatch, + ): + claude_path = tmp_path / "claude" + claude_path.write_text( + """#!/bin/sh +cat > final_report.json <<'JSON' +{ + "success": true, + "run_id": "6e578748-d597-4fde-a3a4-203587cad5a2", + "run_name": "qwen-agent-candidate", + "service_yaml": "type: service\\nname: qwen-agent-candidate\\n", + "verification_summary": "Verified chat completions over SSH." +} +JSON +printf '%s\\n' '{"type":"result","is_error":false,"result":"done"}' +""", + encoding="utf-8", + ) + claude_path.chmod(0o755) + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", str(claude_path)) + (tmp_path / "work").mkdir() + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=None, + env={}, + redacted_values=[], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + max_agent_budget=None, + ) + request = { + "prompt": "prompt", + "env": {}, + "cwd": str(tmp_path / "work"), + "options": { + "allowed_tools": "Bash", + "disallowed_tools": "", + "model": "test-model", + "max_turns": 1, + "max_budget": None, + "json_schema": {}, + }, + } + + result = await _run_agent_in_subprocess(workspace, request) + + assert result.error is None + assert result.report is not None + assert result.report.success is True + assert result.report.run_name == "qwen-agent-candidate" + assert result.report.verification_summary == "Verified chat completions over SSH." + + @pytest.mark.asyncio + async def test_subprocess_no_report_error_does_not_include_stream_output( + self, + tmp_path, + monkeypatch, + ): + claude_path = tmp_path / "claude" + claude_path.write_text( + "#!/bin/sh\n" + 'printf \'%s\\n\' \'{"type":"user","message":{"content":[{"type":"tool_result","content":"huge offer table","is_error":false}]}}\' >&2\n' + "exit 143\n", + encoding="utf-8", + ) + claude_path.chmod(0o755) + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", str(claude_path)) + (tmp_path / "work").mkdir() + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=None, + env={}, + redacted_values=[], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + max_agent_budget=None, + ) + request = { + "prompt": "prompt", + "env": {}, + "options": { + "allowed_tools": "Bash", + "disallowed_tools": "", + "model": "test-model", + "max_turns": 1, + "max_budget": None, + "json_schema": {}, + }, + } + + result = await _run_agent_in_subprocess(workspace, request) + + assert result.error == ( + "Server agent process exited without a verification report (return code 143)" + ) + assert "huge offer table" not in result.error diff --git a/src/tests/_internal/server/services/test_endpoint_presets.py b/src/tests/_internal/server/services/test_endpoint_presets.py new file mode 100644 index 0000000000..45e5321eb5 --- /dev/null +++ b/src/tests/_internal/server/services/test_endpoint_presets.py @@ -0,0 +1,784 @@ +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.configurations import ServiceConfiguration +from dstack._internal.core.models.endpoints import EndpointConfiguration +from dstack._internal.core.models.envs import Env +from dstack._internal.core.models.instances import ( + Disk, + Gpu, + InstanceAvailability, + InstanceOfferWithAvailability, + InstanceRuntime, + InstanceType, + Resources, +) +from dstack._internal.core.models.profiles import CreationPolicy +from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus +from dstack._internal.core.models.users import GlobalRole +from dstack._internal.server.services.endpoints.names import get_endpoint_serving_run_name +from dstack._internal.server.services.endpoints.planning import ( + build_preset_run_spec, + build_preset_service_configuration, + find_matching_preset_plan, + find_preset_planning_result, +) +from dstack._internal.server.services.endpoints.preset_building import ( + build_endpoint_preset_from_run, +) +from dstack._internal.server.services.endpoints.presets import ( + EndpointPreset, + EndpointPresetReplicaSpecGroup, + LocalDirEndpointPresetService, +) +from dstack._internal.server.testing.common import ( + create_job, + create_project, + create_repo, + create_run, + create_user, + get_job_runtime_data, +) + + +class TestLocalDirEndpointPresetService: + @pytest.mark.asyncio + async def test_lists_valid_endpoint_presets(self, tmp_path): + (tmp_path / "qwen.dstack.yml").write_text( + """\ +model: Qwen/Qwen3-4B +type: endpoint-preset +service: + image: vllm/vllm-openai:latest + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 +replica_spec_groups: + - replica_specs: + - gpu: 16GB +""" + ) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + + assert len(presets) == 1 + assert presets[0].name == "qwen" + assert presets[0].model == "Qwen/Qwen3-4B" + assert len(presets[0].replica_spec_groups) == 1 + assert presets[0].replica_spec_groups[0].name == "0" + assert presets[0].replica_spec_groups[0].replica_specs[0].gpu is not None + assert presets[0].configuration.resources.gpu is not None + + @pytest.mark.asyncio + async def test_lists_replica_group_presets_in_group_order(self, tmp_path): + (tmp_path / "qwen.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + port: 8000 + model: Qwen/Qwen3-4B + replicas: + - name: router + count: 1 + image: ghcr.io/example/router:latest + commands: + - python router.py + - name: worker + count: 2 + image: vllm/vllm-openai:latest + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 +replica_spec_groups: + - name: router + replica_specs: + - gpu: 16GB + - name: worker + replica_specs: + - gpu: 24GB + - gpu: 24GB +""" + ) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + + assert len(presets) == 1 + assert [group.name for group in presets[0].replica_spec_groups] == ["router", "worker"] + replica_groups = presets[0].configuration.replica_groups + assert [group.name for group in replica_groups] == ["router", "worker"] + assert all(group.resources.gpu is not None for group in replica_groups) + + @pytest.mark.asyncio + async def test_saves_preset_without_overwriting(self, tmp_path): + preset = _qwen_preset(name="qwen") + service = LocalDirEndpointPresetService(tmp_path) + + saved = await service.save_preset( + preset, + comments=[ + "endpoint: qwen-endpoint", + "run: 00000000-0000-0000-0000-000000000001", + ], + ) + saved_again = await service.save_preset(preset) + + assert saved.name == "qwen" + assert saved_again.name == "qwen-2" + assert (tmp_path / "qwen.dstack.yml").exists() + assert (tmp_path / "qwen-2.dstack.yml").exists() + text = (tmp_path / "qwen.dstack.yml").read_text() + assert text.startswith( + "# endpoint: qwen-endpoint\n# run: 00000000-0000-0000-0000-000000000001\n" + ) + assert "SECRET_VALUE" not in text + assert "preset-service-name" not in text + assert "creation_policy" not in text + assert "resources:" not in text + assert "HF_HOME=/root/.cache/huggingface" in text + assert "HF_TOKEN" in text + + @pytest.mark.asyncio + async def test_saved_preset_round_trips(self, tmp_path): + saved = await LocalDirEndpointPresetService(tmp_path).save_preset( + _qwen_preset(name="Qwen/Qwen3-4B vLLM") + ) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + + assert saved.name == "qwen-qwen3-4b-vllm" + assert len(presets) == 1 + assert presets[0].name == saved.name + assert presets[0].model == "Qwen/Qwen3-4B" + assert presets[0].configuration.model is not None + assert presets[0].configuration.model.name == "Qwen/Qwen3-4B" + assert set(presets[0].configuration.env.keys()) == {"HF_HOME", "HF_TOKEN"} + assert presets[0].configuration.env["HF_HOME"] == "/root/.cache/huggingface" + assert presets[0].configuration.resources.gpu is not None + + @pytest.mark.asyncio + async def test_skips_invalid_presets(self, tmp_path): + (tmp_path / "task.yml").write_text("type: task\ncommands:\n - echo nope\n") + (tmp_path / "missing-model.yml").write_text( + """\ +type: service +commands: + - python -m http.server 8000 +port: 8000 +""" + ) + (tmp_path / "mixed-resources.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + commands: + - python -m http.server 8000 + port: 8000 + resources: + gpu: 24GB +replica_spec_groups: + - replica_specs: + - gpu: 16GB +""" + ) + (tmp_path / "service-profile.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + commands: + - python -m http.server 8000 + port: 8000 + creation_policy: reuse +replica_spec_groups: + - replica_specs: + - gpu: 16GB +""" + ) + (tmp_path / "group-resources.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + port: 8000 + replicas: + - name: worker + count: 1 + image: vllm/vllm-openai:latest + resources: + gpu: 24GB +replica_spec_groups: + - name: worker + replica_specs: + - gpu: 16GB +""" + ) + (tmp_path / "missing-replica-spec-groups.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + port: 8000 + replicas: + - name: worker + count: 1 + image: vllm/vllm-openai:latest +""" + ) + (tmp_path / "mismatched-replica-spec-groups.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + port: 8000 + replicas: + - name: worker + count: 1 + image: vllm/vllm-openai:latest +replica_spec_groups: + - name: other + replica_specs: + - gpu: 16GB +""" + ) + (tmp_path / "heterogeneous-group.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + port: 8000 + replicas: + - name: worker + count: 2 + image: vllm/vllm-openai:latest +replica_spec_groups: + - name: worker + replica_specs: + - gpu: 16GB + - gpu: 24GB +""" + ) + (tmp_path / "bad-replica-group-shape.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + port: 8000 + replicas: + - worker +replica_spec_groups: + - name: worker + replica_specs: + - gpu: 16GB +""" + ) + (tmp_path / "not-yaml.yml").write_text(":\n") + (tmp_path / "notes.txt").write_text("ignored") + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + + assert presets == [] + + +class TestBuildEndpointPresetFromRun: + @pytest.mark.asyncio + async def test_builds_implicit_replica_group_from_running_service( + self, session: AsyncSession, tmp_path + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + run_name="qwen-endpoint", + status=RunStatus.RUNNING, + run_spec=RunSpec( + run_name="qwen-endpoint", + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint", + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-4B", + "resources": {"gpu": "16GB"}, + } + ), + ), + ) + await create_job( + session=session, + run=run, + status=JobStatus.RUNNING, + registered=True, + job_runtime_data=get_job_runtime_data( + offer=_instance_offer(gpu_name="L4", gpu_memory_gib=24, cpu_count=14) + ), + ) + await session.refresh(run, attribute_names=["jobs"]) + + preset = build_endpoint_preset_from_run("qwen-learned", run) + saved = await LocalDirEndpointPresetService(tmp_path).save_preset(preset) + + assert saved.name == "qwen-learned" + assert saved.model == "Qwen/Qwen3-4B" + assert [group.name for group in saved.replica_spec_groups] == ["0"] + resources = saved.replica_spec_groups[0].replica_specs[0].pretty_format() + assert "cpu=14" in resources + assert "gpu=L4:24GB:1" in resources + + @pytest.mark.asyncio + async def test_builds_replica_groups_in_service_order(self, session: AsyncSession, tmp_path): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + run_name="qwen-endpoint", + status=RunStatus.RUNNING, + run_spec=RunSpec( + run_name="qwen-endpoint", + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint", + "port": 8000, + "model": "Qwen/Qwen3-4B", + "replicas": [ + { + "name": "router", + "count": 1, + "commands": ["python router.py"], + "resources": {"cpu": 4}, + }, + { + "name": "worker", + "count": 2, + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "resources": {"gpu": "24GB"}, + }, + ], + } + ), + ), + ) + await create_job( + session=session, + run=run, + status=JobStatus.RUNNING, + registered=True, + replica_num=0, + replica_group_name="router", + job_runtime_data=get_job_runtime_data(offer=_instance_offer(cpu_count=8, gpu_count=0)), + ) + await create_job( + session=session, + run=run, + status=JobStatus.RUNNING, + registered=True, + replica_num=1, + replica_group_name="worker", + job_runtime_data=get_job_runtime_data( + offer=_instance_offer(gpu_name="L4", gpu_memory_gib=24) + ), + ) + await create_job( + session=session, + run=run, + status=JobStatus.RUNNING, + registered=True, + replica_num=2, + replica_group_name="worker", + job_runtime_data=get_job_runtime_data( + offer=_instance_offer(gpu_name="L4", gpu_memory_gib=24) + ), + ) + await session.refresh(run, attribute_names=["jobs"]) + + preset = build_endpoint_preset_from_run("qwen-grouped", run) + saved = await LocalDirEndpointPresetService(tmp_path).save_preset(preset) + + assert [group.name for group in saved.replica_spec_groups] == ["router", "worker"] + assert len(saved.replica_spec_groups[0].replica_specs) == 1 + assert len(saved.replica_spec_groups[1].replica_specs) == 2 + assert [group.name for group in saved.configuration.replica_groups] == [ + "router", + "worker", + ] + + +class TestBuildPresetServiceConfiguration: + def test_get_endpoint_serving_run_name(self): + assert get_endpoint_serving_run_name("qwen-endpoint") == "qwen-endpoint-serving" + + def test_get_endpoint_serving_run_name_keeps_long_name_to_avoid_truncation(self): + endpoint_name = "a" * 41 + + assert get_endpoint_serving_run_name(endpoint_name) == endpoint_name + + def test_endpoint_name_env_and_profile_are_applied(self): + preset = EndpointPreset( + name="qwen", + model="Qwen/Qwen3-4B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "0", "replica_specs": [{"gpu": "16GB"}]} + ) + ], + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "preset-name", + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-4B", + "env": {"HF_HOME": "/root/.cache/huggingface", "HF_TOKEN": "preset"}, + "resources": {"gpu": "16GB"}, + } + ), + ) + endpoint_configuration = EndpointConfiguration( + name="endpoint-name", + model="Qwen/Qwen3-4B", + env=Env.parse_obj({"HF_TOKEN": "endpoint"}), + creation_policy=CreationPolicy.REUSE_OR_CREATE, + max_price=1.5, + ) + + service_configuration = build_preset_service_configuration( + endpoint_name="endpoint-name", + endpoint_configuration=endpoint_configuration, + preset=preset, + ) + + assert service_configuration.name == "endpoint-name-serving" + assert service_configuration.env.as_dict() == { + "HF_HOME": "/root/.cache/huggingface", + "HF_TOKEN": "endpoint", + } + assert service_configuration.creation_policy == CreationPolicy.REUSE_OR_CREATE + assert service_configuration.max_price == 1.5 + assert service_configuration.resources.gpu is not None + + def test_builds_repo_less_run_spec(self): + preset = EndpointPreset( + name="qwen", + model="Qwen/Qwen3-4B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "0", "replica_specs": [{"gpu": "16GB"}]} + ) + ], + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-4B", + } + ), + ) + endpoint_configuration = EndpointConfiguration( + name="endpoint-name", + model="Qwen/Qwen3-4B", + ) + + run_spec = build_preset_run_spec( + endpoint_name="endpoint-name", + endpoint_configuration=endpoint_configuration, + preset=preset, + ) + + assert run_spec.run_name == "endpoint-name-serving" + assert run_spec.repo_id is None + assert run_spec.repo_data is None + assert run_spec.ssh_key_pub is None + assert run_spec.configuration.name == "endpoint-name-serving" + + @pytest.mark.asyncio + async def test_missing_dir_returns_no_presets(self, tmp_path): + presets = await LocalDirEndpointPresetService(tmp_path / "missing").list_presets() + + assert presets == [] + + +class TestFindMatchingPreset: + @pytest.mark.asyncio + async def test_returns_first_preset_with_available_offers( + self, session: AsyncSession, tmp_path + ): + _write_qwen_preset(tmp_path) + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + endpoint_configuration = EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(return_value=_run_plan_with_offer(available=True)), + ) as get_plan_mock: + match = await find_matching_preset_plan( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=endpoint_configuration, + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert match is not None + assert match.preset.name == "qwen" + get_plan_mock.assert_awaited_once() + assert get_plan_mock.await_args is not None + assert get_plan_mock.await_args.kwargs["run_spec"].run_name == "qwen-endpoint-serving" + + @pytest.mark.asyncio + async def test_tracks_first_unprovisionable_preset_when_offers_are_unavailable( + self, session: AsyncSession, tmp_path + ): + _write_qwen_preset(tmp_path) + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(return_value=_run_plan_with_offer(available=False)), + ): + result = await find_preset_planning_result( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert result.provisionable is None + assert result.unprovisionable is not None + assert result.unprovisionable.preset.name == "qwen" + + @pytest.mark.asyncio + async def test_matching_plan_requires_available_offers(self, session: AsyncSession, tmp_path): + _write_qwen_preset(tmp_path) + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(return_value=_run_plan_with_offer(available=False)), + ): + match = await find_matching_preset_plan( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert match is None + + @pytest.mark.asyncio + async def test_skips_preset_with_unresolved_env_sentinel( + self, session: AsyncSession, tmp_path + ): + (tmp_path / "qwen.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + env: + - HF_TOKEN + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 +replica_spec_groups: + - replica_specs: + - gpu: 16GB +""" + ) + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(), + ) as get_plan_mock: + result = await find_preset_planning_result( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert result.provisionable is None + assert result.unprovisionable is None + get_plan_mock.assert_not_awaited() + + @pytest.mark.asyncio + async def test_refreshes_user_ssh_key_before_planning(self, session: AsyncSession, tmp_path): + _write_qwen_preset(tmp_path) + user = await create_user(session=session, global_role=GlobalRole.USER) + project = await create_project(session=session, owner=user) + user.ssh_public_key = None + await session.commit() + + with ( + patch( + "dstack._internal.server.services.endpoints.planning.users_services.refresh_ssh_key", + new=AsyncMock(return_value=user), + ) as refresh_mock, + patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(return_value=_run_plan_with_offer(available=True)), + ), + ): + match = await find_matching_preset_plan( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert match is not None + refresh_mock.assert_awaited_once() + + +def _write_qwen_preset(tmp_path): + (tmp_path / "qwen.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 +replica_spec_groups: + - replica_specs: + - gpu: 16GB +""" + ) + + +def _instance_offer( + gpu_name: str = "T4", + gpu_memory_gib: float = 16, + gpu_count: int = 1, + cpu_count: int = 4, + memory_gib: float = 16, + disk_gib: float = 100, +) -> InstanceOfferWithAvailability: + return InstanceOfferWithAvailability( + backend=BackendType.AWS, + instance=InstanceType( + name="test-instance", + resources=Resources( + cpus=cpu_count, + memory_mib=int(memory_gib * 1024), + gpus=[ + Gpu( + name=gpu_name, + memory_mib=int(gpu_memory_gib * 1024), + ) + for _ in range(gpu_count) + ], + spot=False, + disk=Disk(size_mib=int(disk_gib * 1024)), + ), + ), + region="us-east-1", + price=1.0, + availability=InstanceAvailability.AVAILABLE, + instance_runtime=InstanceRuntime.SHIM, + ) + + +def _qwen_preset(name: str) -> EndpointPreset: + return EndpointPreset( + name=name, + model="Qwen/Qwen3-4B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "0", "replica_specs": [{"gpu": "16GB"}]} + ) + ], + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "preset-service-name", + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-4B", + "env": { + "HF_HOME": "/root/.cache/huggingface", + "HF_TOKEN": "SECRET_VALUE", + }, + "resources": {"gpu": "16GB"}, + "creation_policy": "reuse", + } + ), + ) + + +def _run_plan_with_offer(available: bool): + availability = Mock() + availability.is_available.return_value = available + offer = Mock(availability=availability) + job_plan = Mock(offers=[offer]) + job_plan.job_spec.requirements.resources = Mock() + job_plan.job_spec.requirements.spot = None + job_plan.job_spec.requirements.max_price = None + return Mock(job_plans=[job_plan]) From e1b3e0995cf5ca23642a20f746873a1aa1111688 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 17:01:14 +0200 Subject: [PATCH 02/21] Record endpoint checkpoint cleanup --- endpoint-agent-checkpoints.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index 516909b54e..ac87a10c80 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -97,7 +97,6 @@ These are outside the repo and are not part of the commit: ### Known Issues / Next Hardening -- The endpoint service `qwen-happy-v2` may still be running and spending `$0.44/hr`. - Server logs can still become noisy when agent output contains large YAML/CLI output. - Endpoint logs now resolve to the backing service, but debug trace storage and readable endpoint-agent logs need more real-run pressure. @@ -113,3 +112,21 @@ These are outside the repo and are not part of the commit: This checkpoint should save the code version and the first working evidence. It should not be treated as v1 quality. The point is to keep a recoverable version while the real agent loop evolves through repeated tests. + +### Post-Checkpoint Cleanup + +After the checkpoint commit and tag were created, the live endpoint was deleted to stop +GPU spend: + +```bash +uv run dstack endpoint delete qwen-endpoint-happy -y +uv run dstack run get qwen-happy-v2 --json +uv run dstack endpoint get qwen-endpoint-happy --json +``` + +Observed result on 2026-07-04: + +- `dstack endpoint delete qwen-endpoint-happy -y` returned `Endpoint qwen-endpoint-happy deleted`. +- The backing service moved to `terminating`, then `dstack run get qwen-happy-v2 --json` + returned `Run qwen-happy-v2 not found`. +- `dstack endpoint get qwen-endpoint-happy --json` returned `Endpoint not found`. From 5d5941851d3c45389d8c5cd717e583647841edd0 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 17:05:45 +0200 Subject: [PATCH 03/21] Prefer preset reuse in endpoint smoke config --- endpoint-agent-checkpoints.md | 18 ++++++++++++++++++ qwen-endpoint-happy.dstack.yml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index ac87a10c80..a43e8ea0db 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -130,3 +130,21 @@ Observed result on 2026-07-04: - The backing service moved to `terminating`, then `dstack run get qwen-happy-v2 --json` returned `Run qwen-happy-v2 not found`. - `dstack endpoint get qwen-endpoint-happy --json` returned `Endpoint not found`. + +### Preset Reuse Preview + +After cleanup, `qwen-endpoint-happy.dstack.yml` was changed from `preset_policy: create` +to `preset_policy: reuse-or-create` so rerunning the smoke uses the saved preset first. + +Preview command: + +```bash +echo n | uv run dstack apply -f qwen-endpoint-happy.dstack.yml +``` + +Observed result on 2026-07-04: + +- Matched preset: `qwen-qwen3-0-6b-b0831bdc`. +- Planned resources: `cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1`. +- Planned offer: RunPod `CA-MTL-1`, NVIDIA A40, `$0.44/hr`. +- The command exited at the confirmation prompt; no endpoint or GPU run was created. diff --git a/qwen-endpoint-happy.dstack.yml b/qwen-endpoint-happy.dstack.yml index dedf38ae67..40a1493aa5 100644 --- a/qwen-endpoint-happy.dstack.yml +++ b/qwen-endpoint-happy.dstack.yml @@ -2,7 +2,7 @@ type: endpoint name: qwen-endpoint-happy model: Qwen/Qwen3-0.6B -preset_policy: create +preset_policy: reuse-or-create max_agent_budget: 2 spot_policy: on-demand From ed52172c0ac79d15e68b34eb3d2a013a10236ad7 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 17:13:29 +0200 Subject: [PATCH 04/21] Show latest finished endpoint in watch mode --- src/dstack/_internal/cli/commands/endpoint.py | 4 +--- src/tests/_internal/cli/utils/test_endpoint.py | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py index 28d58f53b2..153904ca15 100644 --- a/src/dstack/_internal/cli/commands/endpoint.py +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -40,8 +40,7 @@ def _register(self): "--all", help=( "Show all endpoints. By default, it only shows unfinished endpoints " - "and the last finished endpoint. In watch mode, it only shows " - "unfinished endpoints." + "and the last finished endpoint." ), action="store_true", ) @@ -117,7 +116,6 @@ def _get_endpoints_for_listing(self, args: argparse.Namespace): endpoints, show_all=args.all, limit=args.last, - include_latest_finished=not args.watch, ) def _delete(self, args: argparse.Namespace): diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py index 260c04b1b8..46e2991187 100644 --- a/src/tests/_internal/cli/utils/test_endpoint.py +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -111,7 +111,7 @@ def test_default_shows_latest_finished_when_none_are_unfinished(self): assert [endpoint.name for endpoint in filtered] == ["failed-new"] - def test_watch_default_shows_only_unfinished(self): + def test_default_shows_latest_finished_and_unfinished_in_watch(self): endpoints = [ _get_endpoint( name="failed-new", @@ -125,9 +125,9 @@ def test_watch_default_shows_only_unfinished(self): ), ] - filtered = filter_endpoints_for_listing(endpoints, include_latest_finished=False) + filtered = filter_endpoints_for_listing(endpoints) - assert [endpoint.name for endpoint in filtered] == ["agenting"] + assert [endpoint.name for endpoint in filtered] == ["agenting", "failed-new"] def test_all_shows_all_sorted_newest_first(self): endpoints = [ From 547b7e997ed5080f04f009bc68057309bee65b09 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 17:15:36 +0200 Subject: [PATCH 05/21] Keep endpoint agent status messages compact --- endpoint-agent-checkpoints.md | 18 ++++++ .../background/pipeline_tasks/endpoints.py | 20 +++++- .../pipeline_tasks/test_endpoints.py | 63 +++++++++++++++++++ 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index a43e8ea0db..8f62f4d0b2 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -148,3 +148,21 @@ Observed result on 2026-07-04: - Planned resources: `cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1`. - Planned offer: RunPod `CA-MTL-1`, NVIDIA A40, `$0.44/hr`. - The command exited at the confirmation prompt; no endpoint or GPU run was created. + +### Agent Status Message Boundary + +Endpoint agent failures now keep endpoint `status_message` compact: agent-originated +errors and failure summaries are collapsed to one line and capped at 500 characters. +Full details remain in the endpoint agent workspace artifacts and endpoint logs. + +Verification on 2026-07-04: + +```bash +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/server/background/pipeline_tasks/endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +``` + +Observed result: + +- endpoint pytest slice: `112 passed, 44 skipped` +- ruff: `All checks passed!` diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py index cac81aa8a1..cf760f89ae 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -61,6 +61,7 @@ logger = get_logger(__name__) _NO_MATCHING_PRESET_MESSAGE = "No matching endpoint presets found." +_MAX_AGENT_STATUS_MESSAGE_CHARS = 500 @dataclass @@ -538,7 +539,7 @@ async def _provision_endpoint_with_agent( return _ProcessResult( update_map={ "status": EndpointStatus.FAILED, - "status_message": result.error, + "status_message": _format_agent_status_message(result.error), } ) report = result.final_report @@ -553,8 +554,9 @@ async def _provision_endpoint_with_agent( return _ProcessResult( update_map={ "status": EndpointStatus.FAILED, - "status_message": ( - report.failure_summary or "Server agent did not verify the endpoint" + "status_message": _format_agent_status_message( + report.failure_summary, + default="Server agent did not verify the endpoint", ), } ) @@ -650,6 +652,18 @@ async def _provision_endpoint_with_agent( ) +def _format_agent_status_message( + message: Optional[str], + default: str = "Server agent failed", +) -> str: + if message is None or not message.strip(): + return default + one_line_message = " ".join(message.split()) + if len(one_line_message) <= _MAX_AGENT_STATUS_MESSAGE_CHARS: + return one_line_message + return one_line_message[: _MAX_AGENT_STATUS_MESSAGE_CHARS - 3].rstrip() + "..." + + async def _record_endpoint_run_submission(endpoint_id: uuid.UUID, run_id: uuid.UUID) -> None: async with get_session_ctx() as session: await record_endpoint_run_submission( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index e73c7f008e..1169876024 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -971,6 +971,69 @@ async def test_agent_failure_fails_endpoint( assert endpoint_model.status_message == "agent could not find a deployable recipe" agent_service.provision_endpoint.assert_awaited_once() + async def test_agent_error_status_message_is_compact( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + long_error = "agent failed before report\n" + "\n".join( + f"offer {i:04d} gpu=A5000 price=0.27" for i in range(200) + ) + agent_service = _FakeAgentService(result=AgentProvisioningResult(error=long_error)) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message is not None + assert len(endpoint_model.status_message) <= 500 + assert "\n" not in endpoint_model.status_message + assert endpoint_model.status_message.startswith("agent failed before report offer 0000") + assert "offer 0199" not in endpoint_model.status_message + + async def test_agent_failure_report_status_message_is_compact( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + long_failure_summary = "agent could not verify service\n" + "\n".join( + f"line {i:04d} with detailed output" for i in range(200) + ) + agent_service = _FakeAgentService( + result=AgentProvisioningResult( + final_report=AgentFinalReport( + success=False, + failure_summary=long_failure_summary, + ) + ) + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message is not None + assert len(endpoint_model.status_message) <= 500 + assert "\n" not in endpoint_model.status_message + assert endpoint_model.status_message.startswith("agent could not verify service line 0000") + assert "line 0199" not in endpoint_model.status_message + async def test_agent_failure_does_not_stop_unlinked_same_name_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): From 3b1a1a9123d26d348b586578e218933b5de28557 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 18:15:53 +0200 Subject: [PATCH 06/21] Show endpoint failure reasons in status --- endpoint-agent-checkpoints.md | 28 ++++++++ src/dstack/_internal/cli/utils/endpoint.py | 71 ++++++++++++++++--- .../_internal/cli/utils/test_endpoint.py | 55 +++++++++++++- 3 files changed, 143 insertions(+), 11 deletions(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index 8f62f4d0b2..1f5e37670d 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -166,3 +166,31 @@ Observed result: - endpoint pytest slice: `112 passed, 44 skipped` - ruff: `All checks passed!` + +### Endpoint Failure Status UX + +Endpoint tables now show short failure reasons in the `STATUS` column, similar to +`dstack ps`, instead of always showing only `failed`. + +Examples observed locally: + +- `no offers` for an agent-confirmed no-offers/no-capacity endpoint +- `agent failed` for failed agent runs without a verified report +- `no agent` when preset creation requires the server agent but the runtime/key is missing + +The verbose `ERROR` column still contains the endpoint status message. + +Verification on 2026-07-04: + +```bash +uv run pytest src/tests/_internal/cli/utils/test_endpoint.py +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/cli/utils/endpoint.py src/tests/_internal/cli/utils/test_endpoint.py +uv run dstack endpoint -a +``` + +Observed result: + +- endpoint utility pytest: `14 passed` +- broader endpoint pytest: `116 passed, 44 skipped` +- ruff: `All checks passed!` diff --git a/src/dstack/_internal/cli/utils/endpoint.py b/src/dstack/_internal/cli/utils/endpoint.py index e5c8acaa2f..d400f5dd0b 100644 --- a/src/dstack/_internal/cli/utils/endpoint.py +++ b/src/dstack/_internal/cli/utils/endpoint.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from rich.table import Table @@ -51,7 +51,7 @@ def get_endpoints_table( table = Table(box=None) table.add_column("NAME", no_wrap=True) table.add_column("MODEL") - table.add_column("STATUS") + table.add_column("STATUS", no_wrap=True) table.add_column("RUN") if verbose: table.add_column("URL") @@ -63,7 +63,7 @@ def get_endpoints_table( row = { "NAME": endpoint.name, "MODEL": endpoint.configuration.model, - "STATUS": _format_endpoint_status(endpoint.status), + "STATUS": _format_endpoint_status(endpoint.status, endpoint.status_message), "RUN": endpoint.run_name or "-", "URL": endpoint.url or "-", "CREATED": format_date(endpoint.created_at), @@ -73,7 +73,11 @@ def get_endpoints_table( return table -def _format_endpoint_status(status: EndpointStatus) -> str: +def _format_endpoint_status( + status: EndpointStatus, + status_message: Optional[str] = None, +) -> str: + status_value = _get_endpoint_status_value(status, status_message) color_map = { EndpointStatus.SUBMITTED: "grey", EndpointStatus.PROVISIONING: "deep_sky_blue1", @@ -82,10 +86,61 @@ def _format_endpoint_status(status: EndpointStatus) -> str: EndpointStatus.ACTIVE: "sea_green3", EndpointStatus.FAILED: "indian_red1", } - style = color_map.get(status, "white") + if status_value == "no offers": + style = "gold1" + else: + style = color_map.get(status, "white") if not status.is_finished(): style = f"bold {style}" - status_value = ( - EndpointStatus.RUNNING.value if status == EndpointStatus.ACTIVE else status.value - ) return f"[{style}]{status_value}[/]" + + +def _get_endpoint_status_value( + status: EndpointStatus, + status_message: Optional[str], +) -> str: + if status == EndpointStatus.ACTIVE: + return EndpointStatus.RUNNING.value + if status == EndpointStatus.FAILED: + failure_reason = _get_endpoint_failure_reason(status_message) + if failure_reason is not None: + return failure_reason + return status.value + + +def _get_endpoint_failure_reason(status_message: Optional[str]) -> Optional[str]: + if status_message is None: + return None + normalized = " ".join(status_message.lower().split()) + if not normalized: + return None + if any( + phrase in normalized + for phrase in [ + "no matching instance offers", + "no matching offers", + "no offers", + "no-offer", + "zero offers", + "total_offers: 0", + ] + ): + return "no offers" + if "no fleets" in normalized: + return "no fleets" + if ( + "requires the server agent" in normalized + or "server agent runtime" in normalized + or "dstack_agent_" in normalized + or "claude executable" in normalized + ): + return "no agent" + if "no matching endpoint presets" in normalized: + return "no preset" + if "server agent" in normalized or "agent" in normalized: + return "agent failed" + if "run name" in normalized and "taken" in normalized: + return "conflict" + if "backing service run" in normalized: + return "run failed" + return None diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py index 46e2991187..1df38e02bf 100644 --- a/src/tests/_internal/cli/utils/test_endpoint.py +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -12,6 +12,7 @@ def _get_endpoint( name: str = "qwen-endpoint", status: EndpointStatus = EndpointStatus.FAILED, + status_message: str | None = "No matching endpoint presets found.", created_at: datetime | None = None, ) -> Endpoint: if created_at is None: @@ -25,7 +26,7 @@ def _get_endpoint( created_at=created_at, last_processed_at=created_at, status=status, - status_message="No matching endpoint presets found.", + status_message=status_message, deleted=False, ) @@ -41,12 +42,60 @@ def test_verbose_table_shows_status_message(self): assert "ERROR" in [column.header for column in table.columns] - def test_status_is_colored(self): - table = get_endpoints_table([_get_endpoint()]) + def test_failed_status_without_reason_is_colored(self): + table = get_endpoints_table([_get_endpoint(status_message=None)]) status_column = next(column for column in table.columns if column.header == "STATUS") assert status_column._cells == ["[indian_red1]failed[/]"] + def test_no_preset_status_is_colored(self): + table = get_endpoints_table([_get_endpoint()]) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[indian_red1]no preset[/]"] + + def test_no_agent_status_is_colored(self): + table = get_endpoints_table( + [ + _get_endpoint( + status_message=( + "No matching endpoint presets found. Creating a preset requires " + "the server agent, but DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + ) + ) + ] + ) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[indian_red1]no agent[/]"] + + def test_no_offers_status_is_colored(self): + table = get_endpoints_table( + [ + _get_endpoint( + status_message=( + "No dstack service could be deployed because max_price matches " + "ZERO offers." + ) + ) + ] + ) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[gold1]no offers[/]"] + + def test_agent_failed_status_is_colored(self): + table = get_endpoints_table( + [ + _get_endpoint( + status_message="Server agent process exited without a verification report" + ) + ] + ) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[indian_red1]agent failed[/]"] + def test_running_status_is_colored(self): table = get_endpoints_table([_get_endpoint(status=EndpointStatus.RUNNING)]) From 3eeaf2f8ff1fdde838545556b1cdb41943bff842 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 18:22:00 +0200 Subject: [PATCH 07/21] Show endpoint details by default --- endpoint-agent-checkpoints.md | 28 ++++++++++++++ src/dstack/_internal/cli/commands/endpoint.py | 7 +++- src/dstack/_internal/cli/utils/endpoint.py | 29 ++++++++++++++ .../_internal/cli/utils/test_endpoint.py | 38 ++++++++++++++++++- 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index 1f5e37670d..81c4d248dc 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -194,3 +194,31 @@ Observed result: - endpoint utility pytest: `14 passed` - broader endpoint pytest: `116 passed, 44 skipped` - ruff: `All checks passed!` + +### Endpoint Get UX + +`dstack endpoint get NAME` now prints a human-readable endpoint detail table by +default. `--json` remains available for scripts and exact API inspection. + +This makes failed endpoints easier to inspect without requiring verbose list output +or JSON parsing. List/watch output stays compact and continues to show short failure +reasons in `STATUS`. + +Verification on 2026-07-04: + +```bash +uv run pytest src/tests/_internal/cli/utils/test_endpoint.py +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/cli/commands/endpoint.py src/dstack/_internal/cli/utils/endpoint.py src/tests/_internal/cli/utils/test_endpoint.py +uv run dstack endpoint get qwen-endpoint-no-offers-schema +uv run dstack endpoint get qwen-endpoint-no-offers-schema --json +uv run dstack endpoint get --help +``` + +Observed result: + +- endpoint utility pytest: `15 passed` +- broader endpoint pytest: `117 passed, 44 skipped` +- ruff: `All checks passed!` +- plain `endpoint get` shows Project/User/Endpoint/Model/Status/Run/URL/Created/Error +- JSON output and help output still work diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py index 153904ca15..0dacbc1415 100644 --- a/src/dstack/_internal/cli/commands/endpoint.py +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -14,6 +14,7 @@ from dstack._internal.cli.utils.endpoint import ( filter_endpoints_for_listing, get_endpoints_table, + print_endpoint, print_endpoints_table, ) from dstack._internal.core.errors import ResourceNotExistsError @@ -86,7 +87,6 @@ def _register(self): get_parser.add_argument( "--json", action="store_true", - required=True, help="Output in JSON format", ) get_parser.set_defaults(subfunc=self._get) @@ -144,4 +144,7 @@ def _get(self, args: argparse.Namespace): console.print("Endpoint not found") exit(1) - print(pydantic_orjson_dumps_with_indent(endpoint.dict(), default=None)) + if args.json: + print(pydantic_orjson_dumps_with_indent(endpoint.dict(), default=None)) + return + print_endpoint(endpoint) diff --git a/src/dstack/_internal/cli/utils/endpoint.py b/src/dstack/_internal/cli/utils/endpoint.py index d400f5dd0b..c1507fce19 100644 --- a/src/dstack/_internal/cli/utils/endpoint.py +++ b/src/dstack/_internal/cli/utils/endpoint.py @@ -43,6 +43,35 @@ def print_endpoints_table(endpoints: List[Endpoint], verbose: bool = False): console.print() +def print_endpoint(endpoint: Endpoint): + console.print(get_endpoint_table(endpoint)) + console.print() + + +def get_endpoint_table( + endpoint: Endpoint, + format_date: DateFormatter = pretty_date, +) -> Table: + table = Table(box=None, show_header=False) + table.add_column(no_wrap=True) + table.add_column() + + def th(value: str) -> str: + return f"[bold]{value}[/bold]" + + table.add_row(th("Project"), endpoint.project_name) + table.add_row(th("User"), endpoint.user) + table.add_row(th("Endpoint"), endpoint.name) + table.add_row(th("Model"), endpoint.configuration.model) + table.add_row(th("Status"), _format_endpoint_status(endpoint.status, endpoint.status_message)) + table.add_row(th("Run"), endpoint.run_name or "-") + table.add_row(th("URL"), endpoint.url or "-") + table.add_row(th("Created"), format_date(endpoint.created_at)) + if endpoint.status_message: + table.add_row(th("Error"), endpoint.status_message) + return table + + def get_endpoints_table( endpoints: List[Endpoint], verbose: bool = False, diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py index 1df38e02bf..3906367a72 100644 --- a/src/tests/_internal/cli/utils/test_endpoint.py +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -1,7 +1,11 @@ from datetime import datetime, timezone from uuid import uuid4 -from dstack._internal.cli.utils.endpoint import filter_endpoints_for_listing, get_endpoints_table +from dstack._internal.cli.utils.endpoint import ( + filter_endpoints_for_listing, + get_endpoint_table, + get_endpoints_table, +) from dstack._internal.core.models.endpoints import ( Endpoint, EndpointConfiguration, @@ -109,6 +113,38 @@ def test_agenting_status_is_colored(self): assert status_column._cells == ["[bold medium_purple1]agenting[/]"] +class TestGetEndpointTable: + def test_shows_endpoint_details(self): + endpoint = _get_endpoint( + status_message="No matching endpoint presets found.", + ) + + table = get_endpoint_table(endpoint, format_date=lambda _: "now") + + assert table.columns[0]._cells == [ + "[bold]Project[/bold]", + "[bold]User[/bold]", + "[bold]Endpoint[/bold]", + "[bold]Model[/bold]", + "[bold]Status[/bold]", + "[bold]Run[/bold]", + "[bold]URL[/bold]", + "[bold]Created[/bold]", + "[bold]Error[/bold]", + ] + assert table.columns[1]._cells == [ + "main", + "test-user", + "qwen-endpoint", + "Qwen/Qwen3-0.6B", + "[indian_red1]no preset[/]", + "-", + "-", + "now", + "No matching endpoint presets found.", + ] + + class TestFilterEndpointsForListing: def test_default_shows_unfinished_and_latest_finished(self): endpoints = [ From ba92cb05bda02e7b6bc9cd6a6c3b2f95d0a5ee28 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 18:29:40 +0200 Subject: [PATCH 08/21] Add endpoint preset listing command --- endpoint-agent-checkpoints.md | 29 ++++++ src/dstack/_internal/cli/commands/preset.py | 66 ++++++++++++++ src/dstack/_internal/cli/main.py | 2 + .../_internal/cli/services/completion.py | 5 ++ src/dstack/_internal/cli/utils/preset.py | 52 +++++++++++ .../_internal/core/models/endpoint_presets.py | 13 +++ .../_internal/server/routers/endpoints.py | 31 +++++++ .../server/schemas/endpoint_presets.py | 7 ++ .../server/services/endpoints/presets.py | 29 ++++++ src/dstack/api/server/__init__.py | 5 ++ src/dstack/api/server/_endpoint_presets.py | 20 +++++ src/tests/_internal/cli/utils/test_preset.py | 47 ++++++++++ .../server/routers/test_endpoints.py | 88 +++++++++++++++++++ .../server/services/test_endpoint_presets.py | 17 ++++ 14 files changed, 411 insertions(+) create mode 100644 src/dstack/_internal/cli/commands/preset.py create mode 100644 src/dstack/_internal/cli/utils/preset.py create mode 100644 src/dstack/_internal/core/models/endpoint_presets.py create mode 100644 src/dstack/_internal/server/schemas/endpoint_presets.py create mode 100644 src/dstack/api/server/_endpoint_presets.py create mode 100644 src/tests/_internal/cli/utils/test_preset.py diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index 81c4d248dc..baab8882a7 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -222,3 +222,32 @@ Observed result: - ruff: `All checks passed!` - plain `endpoint get` shows Project/User/Endpoint/Model/Status/Run/URL/Created/Error - JSON output and help output still work + +### Endpoint Preset CLI + +`dstack preset` now lists endpoint presets saved on the server. `dstack preset list` +is equivalent, and `dstack preset delete NAME` removes a saved endpoint preset after +confirmation. + +This is intentionally limited to list/delete. Creating or updating presets remains part +of the endpoint agent flow, where the agent saves a preset only after verifying the final +service. + +Verification on 2026-07-04: + +```bash +uv run pytest src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/core/models/endpoint_presets.py src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/schemas/endpoint_presets.py src/dstack/api/server/_endpoint_presets.py src/dstack/api/server/__init__.py src/dstack/_internal/server/routers/endpoints.py src/dstack/_internal/cli/utils/preset.py src/dstack/_internal/cli/commands/preset.py src/dstack/_internal/cli/services/completion.py src/dstack/_internal/cli/main.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py +uv run dstack preset --help +uv run dstack preset +uv run dstack --help +``` + +Observed result: + +- focused preset/router pytest: `38 passed, 11 skipped` +- broader endpoint/preset pytest: `125 passed, 46 skipped` +- ruff: `All checks passed!` +- `dstack preset` lists the saved Qwen endpoint presets +- top-level help now includes `preset Manage endpoint presets` diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py new file mode 100644 index 0000000000..0d00678697 --- /dev/null +++ b/src/dstack/_internal/cli/commands/preset.py @@ -0,0 +1,66 @@ +import argparse + +from dstack._internal.cli.commands import APIBaseCommand +from dstack._internal.cli.services.completion import EndpointPresetNameCompleter +from dstack._internal.cli.utils.common import confirm_ask, console +from dstack._internal.cli.utils.preset import print_endpoint_presets_table +from dstack._internal.core.errors import ResourceNotExistsError + + +class PresetCommand(APIBaseCommand): + NAME = "preset" + DESCRIPTION = "Manage endpoint presets" + + def _register(self): + super()._register() + self._parser.set_defaults(subfunc=self._list) + subparsers = self._parser.add_subparsers(dest="action") + + list_parser = subparsers.add_parser( + "list", help="List endpoint presets", formatter_class=self._parser.formatter_class + ) + list_parser.set_defaults(subfunc=self._list) + + delete_parser = subparsers.add_parser( + "delete", + help="Delete endpoint presets", + formatter_class=self._parser.formatter_class, + ) + delete_parser.add_argument( + "name", + help="The name of the endpoint preset", + ).completer = EndpointPresetNameCompleter() # type: ignore[attr-defined] + delete_parser.add_argument( + "-y", "--yes", help="Don't ask for confirmation", action="store_true" + ) + delete_parser.set_defaults(subfunc=self._delete) + + def _command(self, args: argparse.Namespace): + super()._command(args) + args.subfunc(args) + + def _list(self, args: argparse.Namespace): + presets = self.api.client.endpoint_presets.list(self.api.project) + print_endpoint_presets_table(presets) + + def _delete(self, args: argparse.Namespace): + presets = self.api.client.endpoint_presets.list(self.api.project) + if args.name not in {preset.name for preset in presets}: + console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + exit(1) + + if not args.yes and not confirm_ask(f"Delete the endpoint preset [code]{args.name}[/]?"): + console.print("\nExiting...") + return + + try: + with console.status("Deleting endpoint preset..."): + self.api.client.endpoint_presets.delete( + project_name=self.api.project, + names=[args.name], + ) + except ResourceNotExistsError: + console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + exit(1) + + console.print(f"Endpoint preset [code]{args.name}[/] deleted") diff --git a/src/dstack/_internal/cli/main.py b/src/dstack/_internal/cli/main.py index 335f7693f1..ed58dc1360 100644 --- a/src/dstack/_internal/cli/main.py +++ b/src/dstack/_internal/cli/main.py @@ -19,6 +19,7 @@ from dstack._internal.cli.commands.logs import LogsCommand from dstack._internal.cli.commands.metrics import MetricsCommand from dstack._internal.cli.commands.offer import OfferCommand +from dstack._internal.cli.commands.preset import PresetCommand from dstack._internal.cli.commands.project import ProjectCommand from dstack._internal.cli.commands.ps import PsCommand from dstack._internal.cli.commands.run import RunCommand @@ -76,6 +77,7 @@ def main(): GatewayCommand.register(subparsers) InitCommand.register(subparsers) OfferCommand.register(subparsers) + PresetCommand.register(subparsers) LoginCommand.register(subparsers) LogsCommand.register(subparsers) MetricsCommand.register(subparsers) diff --git a/src/dstack/_internal/cli/services/completion.py b/src/dstack/_internal/cli/services/completion.py index d80a211a33..754069cc2c 100644 --- a/src/dstack/_internal/cli/services/completion.py +++ b/src/dstack/_internal/cli/services/completion.py @@ -82,6 +82,11 @@ def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.client.endpoints.list(api.project)] +class EndpointPresetNameCompleter(BaseAPINameCompleter): + def fetch_resource_names(self, api: Client) -> Iterable[str]: + return [r.name for r in api.client.endpoint_presets.list(api.project)] + + class GatewayNameCompleter(BaseAPINameCompleter): def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.client.gateways.list(api.project)] diff --git a/src/dstack/_internal/cli/utils/preset.py b/src/dstack/_internal/cli/utils/preset.py new file mode 100644 index 0000000000..f1d31f3d9a --- /dev/null +++ b/src/dstack/_internal/cli/utils/preset.py @@ -0,0 +1,52 @@ +from typing import List + +from rich.table import Table + +from dstack._internal.cli.utils.common import add_row_from_dict, console +from dstack._internal.core.models.endpoint_presets import EndpointPreset +from dstack._internal.core.models.resources import ResourcesSpec + + +def print_endpoint_presets_table(presets: List[EndpointPreset]): + table = get_endpoint_presets_table(presets) + console.print(table) + console.print() + + +def get_endpoint_presets_table(presets: List[EndpointPreset]) -> Table: + table = Table(box=None) + table.add_column("NAME", style="bold", no_wrap=True) + table.add_column("MODEL") + table.add_column("RESOURCES") + + for preset in presets: + add_row_from_dict( + table, + { + "NAME": preset.name, + "MODEL": preset.model, + "RESOURCES": _format_replica_spec_groups(preset), + }, + ) + return table + + +def _format_replica_spec_groups(preset: EndpointPreset) -> str: + groups = [] + show_group_names = len(preset.replica_spec_groups) > 1 + for group in preset.replica_spec_groups: + value = _format_replica_specs(group.replica_specs) + if show_group_names or group.name != "0": + value = f"{group.name}: {value}" + groups.append(value) + return "; ".join(groups) if groups else "-" + + +def _format_replica_specs(replica_specs: list[ResourcesSpec]) -> str: + formatted_specs = [spec.pretty_format() for spec in replica_specs] + if not formatted_specs: + return "-" + unique_specs = set(formatted_specs) + if len(unique_specs) == 1 and len(formatted_specs) > 1: + return f"{len(formatted_specs)}x {formatted_specs[0]}" + return ", ".join(formatted_specs) diff --git a/src/dstack/_internal/core/models/endpoint_presets.py b/src/dstack/_internal/core/models/endpoint_presets.py new file mode 100644 index 0000000000..558302fa92 --- /dev/null +++ b/src/dstack/_internal/core/models/endpoint_presets.py @@ -0,0 +1,13 @@ +from dstack._internal.core.models.common import CoreModel +from dstack._internal.core.models.resources import ResourcesSpec + + +class EndpointPresetReplicaSpecGroup(CoreModel): + name: str + replica_specs: list[ResourcesSpec] + + +class EndpointPreset(CoreModel): + name: str + model: str + replica_spec_groups: list[EndpointPresetReplicaSpecGroup] diff --git a/src/dstack/_internal/server/routers/endpoints.py b/src/dstack/_internal/server/routers/endpoints.py index 1e1209cd29..b7f891b694 100644 --- a/src/dstack/_internal/server/routers/endpoints.py +++ b/src/dstack/_internal/server/routers/endpoints.py @@ -5,9 +5,11 @@ import dstack._internal.server.services.endpoints as endpoints_services from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.core.models.endpoint_presets import EndpointPreset from dstack._internal.core.models.endpoints import Endpoint, EndpointPlan from dstack._internal.server.db import get_session from dstack._internal.server.models import ProjectModel, UserModel +from dstack._internal.server.schemas.endpoint_presets import DeleteEndpointPresetsRequest from dstack._internal.server.schemas.endpoints import ( CreateEndpointRequest, DeleteEndpointsRequest, @@ -16,6 +18,10 @@ ListEndpointsRequest, ) from dstack._internal.server.security.permissions import Authenticated, ProjectMember +from dstack._internal.server.services.endpoints.presets import ( + endpoint_preset_to_api_model, + get_endpoint_preset_service, +) from dstack._internal.server.services.pipelines import PipelineHinterProtocol, get_pipeline_hinter from dstack._internal.server.utils.routers import ( CustomORJSONResponse, @@ -130,3 +136,28 @@ async def delete_endpoints( user=user, pipeline_hinter=pipeline_hinter, ) + + +@project_router.post( + "/presets/list", summary="List endpoint presets", response_model=List[EndpointPreset] +) +async def list_endpoint_presets( + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), +): + _, _ = user_project + presets = await get_endpoint_preset_service().list_presets() + return CustomORJSONResponse([endpoint_preset_to_api_model(preset) for preset in presets]) + + +@project_router.post("/presets/delete", summary="Delete endpoint presets") +async def delete_endpoint_presets( + body: DeleteEndpointPresetsRequest, + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), +): + _, _ = user_project + preset_service = get_endpoint_preset_service() + try: + for name in body.names: + await preset_service.delete_preset(name) + except FileNotFoundError: + raise ResourceNotExistsError() diff --git a/src/dstack/_internal/server/schemas/endpoint_presets.py b/src/dstack/_internal/server/schemas/endpoint_presets.py new file mode 100644 index 0000000000..92ec58887f --- /dev/null +++ b/src/dstack/_internal/server/schemas/endpoint_presets.py @@ -0,0 +1,7 @@ +from typing import List + +from dstack._internal.core.models.common import CoreModel + + +class DeleteEndpointPresetsRequest(CoreModel): + names: List[str] diff --git a/src/dstack/_internal/server/services/endpoints/presets.py b/src/dstack/_internal/server/services/endpoints/presets.py index ec86d4ec3f..bef7591896 100644 --- a/src/dstack/_internal/server/services/endpoints/presets.py +++ b/src/dstack/_internal/server/services/endpoints/presets.py @@ -15,6 +15,9 @@ DEFAULT_REPLICA_GROUP_NAME, ServiceConfiguration, ) +from dstack._internal.core.models.endpoint_presets import ( + EndpointPreset as EndpointPresetSummary, +) from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.resources import ResourcesSpec @@ -47,6 +50,10 @@ class EndpointPresetService(ABC): async def list_presets(self) -> list[EndpointPreset]: pass + @abstractmethod + async def delete_preset(self, name: str) -> None: + pass + @abstractmethod async def save_preset( self, @@ -64,6 +71,9 @@ def __init__(self, presets_dir: Path = settings.ENDPOINT_PRESETS_DIR) -> None: async def list_presets(self) -> list[EndpointPreset]: return await run_async(self._list_presets) + async def delete_preset(self, name: str) -> None: + return await run_async(self._delete_preset, name) + async def save_preset( self, preset: EndpointPreset, @@ -83,6 +93,17 @@ def _list_presets(self) -> list[EndpointPreset]: presets.append(preset) return presets + def _delete_preset(self, name: str) -> None: + if not self._presets_dir.exists(): + raise FileNotFoundError(name) + for path in self._presets_dir.iterdir(): + if path.suffix not in [".yml", ".yaml"]: + continue + if _get_preset_name_from_path(path) == name: + path.unlink() + return + raise FileNotFoundError(name) + def _load_preset(self, path: Path) -> Optional[EndpointPreset]: try: with path.open("r") as f: @@ -148,6 +169,14 @@ def get_endpoint_preset_service() -> EndpointPresetService: return _endpoint_preset_service +def endpoint_preset_to_api_model(preset: EndpointPreset) -> EndpointPresetSummary: + return EndpointPresetSummary( + name=preset.name, + model=preset.model, + replica_spec_groups=preset.replica_spec_groups, + ) + + def _get_preset_name_from_path(path: Path) -> str: name = path.stem if name.endswith(".dstack"): diff --git a/src/dstack/api/server/__init__.py b/src/dstack/api/server/__init__.py index 5888cee80f..80e779a973 100644 --- a/src/dstack/api/server/__init__.py +++ b/src/dstack/api/server/__init__.py @@ -16,6 +16,7 @@ from dstack._internal.utils.logging import get_logger from dstack.api.server._auth import AuthAPIClient from dstack.api.server._backends import BackendsAPIClient +from dstack.api.server._endpoint_presets import EndpointPresetsAPIClient from dstack.api.server._endpoints import EndpointsAPIClient from dstack.api.server._events import EventsAPIClient from dstack.api.server._exports import ExportsAPIClient @@ -135,6 +136,10 @@ def volumes(self) -> VolumesAPIClient: def endpoints(self) -> EndpointsAPIClient: return EndpointsAPIClient(self._request, self._logger) + @property + def endpoint_presets(self) -> EndpointPresetsAPIClient: + return EndpointPresetsAPIClient(self._request, self._logger) + @property def exports(self) -> ExportsAPIClient: return ExportsAPIClient(self._request, self._logger) diff --git a/src/dstack/api/server/_endpoint_presets.py b/src/dstack/api/server/_endpoint_presets.py new file mode 100644 index 0000000000..bc58c07444 --- /dev/null +++ b/src/dstack/api/server/_endpoint_presets.py @@ -0,0 +1,20 @@ +from typing import List + +from pydantic import parse_obj_as + +from dstack._internal.core.models.endpoint_presets import EndpointPreset +from dstack._internal.server.schemas.endpoint_presets import DeleteEndpointPresetsRequest +from dstack.api.server._group import APIClientGroup + + +class EndpointPresetsAPIClient(APIClientGroup): + def list(self, project_name: str) -> List[EndpointPreset]: + resp = self._request(f"/api/project/{project_name}/endpoints/presets/list") + return parse_obj_as(List[EndpointPreset.__response__], resp.json()) + + def delete(self, project_name: str, names: List[str]) -> None: + body = DeleteEndpointPresetsRequest(names=names) + self._request( + f"/api/project/{project_name}/endpoints/presets/delete", + body=body.json(), + ) diff --git a/src/tests/_internal/cli/utils/test_preset.py b/src/tests/_internal/cli/utils/test_preset.py new file mode 100644 index 0000000000..841de7217c --- /dev/null +++ b/src/tests/_internal/cli/utils/test_preset.py @@ -0,0 +1,47 @@ +from dstack._internal.cli.utils.preset import get_endpoint_presets_table +from dstack._internal.core.models.endpoint_presets import ( + EndpointPreset, + EndpointPresetReplicaSpecGroup, +) + + +class TestGetEndpointPresetsTable: + def test_shows_single_group_preset(self): + preset = EndpointPreset( + name="qwen-qwen3-0-6b", + model="Qwen/Qwen3-0.6B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "0", "replica_specs": [{"gpu": "16GB", "disk": "60GB"}]} + ) + ], + ) + + table = get_endpoint_presets_table([preset]) + + assert table.columns[0]._cells == ["qwen-qwen3-0-6b"] + assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B"] + assert "gpu=16GB" in table.columns[2]._cells[0] + assert "disk=60GB" in table.columns[2]._cells[0] + + def test_shows_grouped_replica_specs(self): + preset = EndpointPreset( + name="qwen-pd", + model="Qwen/Qwen3-30B-A3B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "router", "replica_specs": [{"cpu": 4}]} + ), + EndpointPresetReplicaSpecGroup.parse_obj( + {"name": "worker", "replica_specs": [{"gpu": "24GB"}, {"gpu": "24GB"}]} + ), + ], + ) + + table = get_endpoint_presets_table([preset]) + + resources = table.columns[2]._cells[0] + assert "router:" in resources + assert "worker:" in resources + assert "2x" in resources + assert "gpu=24GB" in resources diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index 6d85360f89..caa91d9f92 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -439,6 +439,94 @@ async def test_marks_endpoint_for_deletion( ] +class TestEndpointPresets: + @pytest.mark.asyncio + async def test_list_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/presets/list") + assert response.status_code in [401, 403] + + @pytest.mark.asyncio + async def test_delete_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/presets/delete") + assert response.status_code in [401, 403] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_lists_endpoint_presets( + self, + test_db, + session: AsyncSession, + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + preset_service = _FakeEndpointPresetService([_endpoint_preset_plan().preset]) + monkeypatch.setattr( + "dstack._internal.server.routers.endpoints.get_endpoint_preset_service", + lambda: preset_service, + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/presets/list", + headers=get_auth_headers(user.token), + ) + + assert response.status_code == 200, response.json() + body = response.json() + assert len(body) == 1 + assert body[0]["name"] == "qwen" + assert body[0]["model"] == "Qwen/Qwen3-0.6B" + assert body[0]["replica_spec_groups"][0]["name"] == "0" + assert body[0]["replica_spec_groups"][0]["replica_specs"][0]["gpu"] is not None + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_deletes_endpoint_preset( + self, + test_db, + session: AsyncSession, + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + preset_service = _FakeEndpointPresetService([_endpoint_preset_plan().preset]) + monkeypatch.setattr( + "dstack._internal.server.routers.endpoints.get_endpoint_preset_service", + lambda: preset_service, + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/presets/delete", + headers=get_auth_headers(user.token), + json={"names": ["qwen"]}, + ) + + assert response.status_code == 200, response.json() + assert preset_service.deleted_names == ["qwen"] + + +class _FakeEndpointPresetService: + def __init__(self, presets): + self._presets = presets + self.deleted_names = [] + + async def list_presets(self): + return self._presets + + async def delete_preset(self, name): + if name not in {preset.name for preset in self._presets}: + raise FileNotFoundError(name) + self.deleted_names.append(name) + + async def _create_endpoint_model( session: AsyncSession, project, diff --git a/src/tests/_internal/server/services/test_endpoint_presets.py b/src/tests/_internal/server/services/test_endpoint_presets.py index 45e5321eb5..d048148a08 100644 --- a/src/tests/_internal/server/services/test_endpoint_presets.py +++ b/src/tests/_internal/server/services/test_endpoint_presets.py @@ -158,6 +158,23 @@ async def test_saved_preset_round_trips(self, tmp_path): assert presets[0].configuration.env["HF_HOME"] == "/root/.cache/huggingface" assert presets[0].configuration.resources.gpu is not None + @pytest.mark.asyncio + async def test_deletes_preset_by_name(self, tmp_path): + service = LocalDirEndpointPresetService(tmp_path) + saved = await service.save_preset(_qwen_preset(name="qwen")) + + await service.delete_preset(saved.name) + + assert await service.list_presets() == [] + assert not (tmp_path / "qwen.dstack.yml").exists() + + @pytest.mark.asyncio + async def test_delete_missing_preset_raises(self, tmp_path): + service = LocalDirEndpointPresetService(tmp_path) + + with pytest.raises(FileNotFoundError): + await service.delete_preset("missing") + @pytest.mark.asyncio async def test_skips_invalid_presets(self, tmp_path): (tmp_path / "task.yml").write_text("type: task\ncommands:\n - echo nope\n") From 3a0209602c946eebbd8406fe21665733df1d3221 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 21:54:15 +0200 Subject: [PATCH 09/21] Checkpoint endpoint agent RunPod smoke --- endpoint-agent-checkpoints.md | 158 +++++++- .../cli/services/configurators/endpoint.py | 4 + src/dstack/_internal/cli/utils/preset.py | 81 ++-- .../_internal/core/models/endpoint_presets.py | 7 +- src/dstack/_internal/core/models/endpoints.py | 7 +- .../server/services/endpoints/__init__.py | 3 +- .../agent/resources/deployment_harness.md | 18 +- .../dstack_cli_and_service_authoring.md | 9 + .../agent/resources/recipes_guide.md | 3 +- .../services/endpoints/preset_building.py | 11 +- .../server/services/endpoints/presets.py | 103 ++++- src/tests/_internal/cli/utils/test_preset.py | 101 ++++- .../pipeline_tasks/test_endpoints.py | 32 ++ .../server/routers/test_endpoints.py | 16 +- .../server/services/test_endpoint_presets.py | 352 ++++++++++++++---- 15 files changed, 756 insertions(+), 149 deletions(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index baab8882a7..6f2cb47df1 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -145,8 +145,9 @@ echo n | uv run dstack apply -f qwen-endpoint-happy.dstack.yml Observed result on 2026-07-04: - Matched preset: `qwen-qwen3-0-6b-b0831bdc`. -- Planned resources: `cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1`. -- Planned offer: RunPod `CA-MTL-1`, NVIDIA A40, `$0.44/hr`. +- Matched preset evidence: one verified RunPod A40 replica. +- Planned resources now come from the preset scheduling requirements, not the exact + verified instance resources. - The command exited at the confirmation prompt; no endpoint or GPU run was created. ### Agent Status Message Boundary @@ -251,3 +252,156 @@ Observed result: - ruff: `All checks passed!` - `dstack preset` lists the saved Qwen endpoint presets - top-level help now includes `preset Manage endpoint presets` + +### Endpoint Preset Resource Contract + +Endpoint presets now separate scheduling requirements from verified runtime evidence: +`replica_spec_groups[*].resources` is used for service planning and offer matching, +while `replica_spec_groups[*].tested_resources` stores exact resources captured from +actual registered service replicas. + +`dstack preset` now displays every actual replica when a preset has multiple replicas, using child rows such as `replica=0` or `group=worker replica=1`, matching the hierarchy used by `dstack ps`. It does not summarize replicas as counts. + +Invalid local preset files are skipped for user-facing preset listing and logged server-side with +the preset path and parse/validation error. + +Verification on 2026-07-04: + +```bash +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/cli/utils/test_preset.py +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/services/endpoints/preset_building.py src/dstack/_internal/cli/utils/preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/cli/utils/test_preset.py +uv run dstack preset +``` + +Observed result: + +- focused preset/endpoint-worker pytest: `69 passed, 35 skipped` +- broader endpoint pytest: `138 passed, 46 skipped` +- ruff: `All checks passed!` +- `dstack preset` skips the old loose smoke preset and lists the valid learned preset; the server logs the skipped preset path and validation error + +### Endpoint Agent Retest After Deleting Preset + +Retest used the current checkout server on `127.0.0.1:3000` with a temporary CLI +home at `/tmp/dstack-endpoint-test-home`; the normal/default CLI config was restored +to `main -> 127.0.0.1:3002`. + +Flow observed on 2026-07-04: + +- Deleted the existing learned Qwen preset. +- Submitted `qwen-endpoint-smoke` with `backend=runpod`, `spot_policy=on-demand`, + `max_price=0.5`. +- Agent created real service candidates and handled real RunPod capacity failures: + `qwen3-06b-smoke` failed on A5000 no-capacity, `qwen3-06b-smoke2` initially retried + the same no-capacity path, then the agent stopped it and tried `qwen3-06b-l4`. +- `qwen3-06b-l4` first tried L4 and then dstack provisioned A40 in CA-MTL-1 at + `$0.44/hr`; vLLM served `Qwen/Qwen3-0.6B` and real `/v1/chat/completions` + requests returned HTTP 200. +- Endpoint reached `running`; learned preset saved as `qwen-qwen3-0-6b-94071a4a`. +- Cleanup completed: endpoint deleted, `qwen3-06b-l4` stopped. + +Important harness findings: + +- Good: real agent loop works end-to-end through deployment, verification, endpoint + running state, and preset save. +- Bad: the agent copied offer/workaround hardware into final service scheduling + requirements (`gpu.name: [L4, A40, RTX3090]`) instead of preserving the broadest + correct model-derived requirement. +- Bad: the agent's verification/final report said L4, but the actual provisioned + hardware and saved `tested_resources` were A40. The server-side preset builder used + actual run state correctly; the agent report was stale/inferred. + +Patch made after this run: + +- Prompt resources now say to derive scheduling requirements from the model/serving + method, treat preview offers as availability evidence rather than target hardware, + avoid pinning concrete GPU/region/instance unless required or explicitly justified, + and re-read `dstack run get --json` after verification to report actual provisioned + hardware. + +## Checkpoint: qwen-runpod-v1-endpoint-dev-running + +Status: known-good endpoint-agent smoke with separate local project and corrected +preset resource contract. + +Expected local tag after commit: + +```bash +endpoint-agent/qwen-runpod-v1-endpoint-dev-running +``` + +Date: 2026-07-04 +Server: current checkout on `127.0.0.1:3000` +CLI project: `endpoint-dev -> http://127.0.0.1:3000` +Default CLI project preserved: `main -> http://127.0.0.1:3002` + +### What Worked + +- Endpoint `qwen-endpoint-smoke` reached `running`. +- Model: `Qwen/Qwen3-0.6B`. +- Agent submitted and verified service run `qwen-smoke`. +- Final run ID: `cfef76ab-9ec7-4c41-913b-e9595e2979cd`. +- Endpoint URL: `/proxy/services/endpoint-dev/qwen-smoke/v1`. +- Backend/hardware: RunPod `EU-RO-1`, NVIDIA RTX 2000 Ada Generation + (`RTX2000Ada:16GB:1`), 6 CPU, 31GB RAM, 100GB disk. +- Hourly price: `$0.24/hr`. +- Agent service YAML used model-derived scheduling requirements: + `resources.gpu: 16GB..`, not a pinned GPU name, region, or instance type. +- Agent used `vllm serve Qwen/Qwen3-0.6B --port 8000 --max-model-len 8192`. +- dstack service probe reached `success_streak: 4`. +- Agent verified both `/v1/models` and `/v1/chat/completions` through the + dstack service proxy with HTTP 200. +- Endpoint preset was saved as `qwen-qwen3-0-6b-cfef76ab`. +- Saved preset uses broad scheduling requirements and exact tested resources: + - scheduling: `cpu=2.. mem=8GB.. disk=100GB.. gpu=16GB..:1..` + - tested: `cpu=6 mem=31GB disk=100GB gpu=RTX2000Ada:16GB:1` + +### Verification Commands + +```bash +uv run dstack endpoint --project endpoint-dev get qwen-endpoint-smoke --json +uv run dstack run --project endpoint-dev get qwen-smoke --json +uv run dstack preset --project endpoint-dev +uv run dstack logs --project endpoint-dev qwen-smoke --since 3m +``` + +Observed result on 2026-07-04: + +- endpoint status: `running` +- backing service status: `running` +- preset listed by `dstack preset --project endpoint-dev` +- `verification.json` recorded HTTP 200 for `/v1/models` and + `/v1/chat/completions` +- `final_report.json` recorded actual provisioned hardware from run JSON + +### Useful Runtime Artifacts + +These are outside the repo and are not part of the checkpoint commit: + +- Final report: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/bb5846b3-4de1-45fb-896f-eaef5ebd73cf/workspace/final_report.json` +- Verification: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/bb5846b3-4de1-45fb-896f-eaef5ebd73cf/workspace/verification.json` +- Saved preset: + `/Users/dstack/.dstack/server/data/endpoint_presets/qwen-qwen3-0-6b-cfef76ab.dstack.yml` + +### Known Issues / Next Hardening + +- After agent verification, the endpoint briefly transitions from `agenting` to + `provisioning` before `running`. This is confusing; the next patch should avoid + exposing that intermediate status for agent-verified endpoints. +- The agent still sometimes uses invalid CLI forms first, such as `dstack run list` + or uppercase backend names. The harness prompt should make the supported command + surface stricter. +- The agent used a generic run name (`qwen-smoke`). Future prompt/harness rules + should require useful unique names for candidate runs. +- The agent used shell polling loops without explicit timeouts. Future rules should + require bounded polling and clear progress notes. +- `dstack logs -d` can expose very verbose shim environment output. Normal endpoint + logs should become a concise major-event stream written by the agent, while full + trace/debug artifacts stay in the workspace. +- The endpoint agent trace is useful for debugging, but it is too detailed for normal + `dstack logs ENDPOINT`. Endpoint logs should contain major realtime events only: + research/plan summary, candidate submitted, provisioning state, service startup, + verification success/failure, preset save, and cleanup. diff --git a/src/dstack/_internal/cli/services/configurators/endpoint.py b/src/dstack/_internal/cli/services/configurators/endpoint.py index 807f291c7e..3a7da0193b 100644 --- a/src/dstack/_internal/cli/services/configurators/endpoint.py +++ b/src/dstack/_internal/cli/services/configurators/endpoint.py @@ -398,7 +398,9 @@ def _print_preset_plan_offers( offers.add_column("PRICE", style="grey58") offers.add_column() offer_num = 0 + total_offers = 0 for job_offers in provisioning_plan.job_offers: + total_offers += job_offers.total_offers for offer in job_offers.offers: offer_num += 1 row = [ @@ -418,6 +420,8 @@ def _print_preset_plan_offers( row.insert(1, job_offers.replica_group) offers.add_row(*row, style="secondary") console.print(offers) + if total_offers > offer_num: + console.print(f"[secondary] Shown {offer_num} of {total_offers} offers[/]") console.print() diff --git a/src/dstack/_internal/cli/utils/preset.py b/src/dstack/_internal/cli/utils/preset.py index f1d31f3d9a..74d98ef123 100644 --- a/src/dstack/_internal/cli/utils/preset.py +++ b/src/dstack/_internal/cli/utils/preset.py @@ -4,7 +4,6 @@ from dstack._internal.cli.utils.common import add_row_from_dict, console from dstack._internal.core.models.endpoint_presets import EndpointPreset -from dstack._internal.core.models.resources import ResourcesSpec def print_endpoint_presets_table(presets: List[EndpointPreset]): @@ -20,33 +19,59 @@ def get_endpoint_presets_table(presets: List[EndpointPreset]) -> Table: table.add_column("RESOURCES") for preset in presets: - add_row_from_dict( - table, - { - "NAME": preset.name, - "MODEL": preset.model, - "RESOURCES": _format_replica_spec_groups(preset), - }, - ) + total_replicas = sum(len(group.tested_resources) for group in preset.replica_spec_groups) + if total_replicas == 1: + add_row_from_dict( + table, + { + "NAME": preset.name, + "MODEL": preset.model, + "RESOURCES": _format_replica_resources( + preset.replica_spec_groups[0].tested_resources[0] + ), + }, + ) + continue + add_row_from_dict(table, {"NAME": preset.name, "MODEL": preset.model}) + show_group = len(preset.replica_spec_groups) > 1 + last_group_index = None + for group_index, group in enumerate(preset.replica_spec_groups): + for replica_num, replica_spec in enumerate(group.tested_resources): + add_row_from_dict( + table, + { + "NAME": _format_replica_name( + group_index=group_index, + group_name=group.name, + replica_num=replica_num, + show_group=show_group, + last_group_index=last_group_index, + ), + "RESOURCES": _format_replica_resources(replica_spec), + }, + ) + last_group_index = group_index return table -def _format_replica_spec_groups(preset: EndpointPreset) -> str: - groups = [] - show_group_names = len(preset.replica_spec_groups) > 1 - for group in preset.replica_spec_groups: - value = _format_replica_specs(group.replica_specs) - if show_group_names or group.name != "0": - value = f"{group.name}: {value}" - groups.append(value) - return "; ".join(groups) if groups else "-" - - -def _format_replica_specs(replica_specs: list[ResourcesSpec]) -> str: - formatted_specs = [spec.pretty_format() for spec in replica_specs] - if not formatted_specs: - return "-" - unique_specs = set(formatted_specs) - if len(unique_specs) == 1 and len(formatted_specs) > 1: - return f"{len(formatted_specs)}x {formatted_specs[0]}" - return ", ".join(formatted_specs) +def _format_replica_name( + *, + group_index: int, + group_name: str, + replica_num: int, + show_group: bool, + last_group_index: int | None, +) -> str: + if not show_group: + return f" replica={replica_num}" + if group_index != last_group_index: + return f" group={group_name} replica={replica_num}" + padding_width = 3 + len(f"group={group_name}") + 1 + return f"{' ' * padding_width}replica={replica_num}" + + +def _format_replica_resources(resources) -> str: + formatted = resources.pretty_format() + if resources.gpu is not None and resources.gpu.count.min == 0 and resources.gpu.count.max == 0: + return formatted.replace(" gpu=0", "").replace("gpu=0", "-") + return formatted diff --git a/src/dstack/_internal/core/models/endpoint_presets.py b/src/dstack/_internal/core/models/endpoint_presets.py index 558302fa92..a6340ec738 100644 --- a/src/dstack/_internal/core/models/endpoint_presets.py +++ b/src/dstack/_internal/core/models/endpoint_presets.py @@ -3,8 +3,13 @@ class EndpointPresetReplicaSpecGroup(CoreModel): + """Ordered to match service replica groups; "0" is the implicit group.""" + name: str - replica_specs: list[ResourcesSpec] + resources: ResourcesSpec + """Per-replica scheduling requirements used when applying the preset.""" + tested_resources: list[ResourcesSpec] + """Exact resources of the replicas that were running when the preset was verified.""" class EndpointPreset(CoreModel): diff --git a/src/dstack/_internal/core/models/endpoints.py b/src/dstack/_internal/core/models/endpoints.py index 09153ad6b8..9d3289c047 100644 --- a/src/dstack/_internal/core/models/endpoints.py +++ b/src/dstack/_internal/core/models/endpoints.py @@ -107,8 +107,13 @@ class EndpointProvisioningPlanNone(CoreModel): class EndpointPlanReplicaSpecGroup(CoreModel): + """Ordered to match service replica groups; "0" is the implicit group.""" + name: str - replica_specs: list[ResourcesSpec] + resources: ResourcesSpec + """Per-replica scheduling requirements used for offer matching.""" + tested_resources: list[ResourcesSpec] + """Exact resources of the replicas that were running when the preset was verified.""" class EndpointPlanJobOffers(CoreModel): diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py index 9f4e05b6db..6d9dac2867 100644 --- a/src/dstack/_internal/server/services/endpoints/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -367,7 +367,8 @@ def _endpoint_preset_plan_to_provisioning_plan( replica_spec_groups=[ EndpointPlanReplicaSpecGroup( name=group.name, - replica_specs=group.replica_specs, + resources=group.resources, + tested_resources=group.tested_resources, ) for group in preset_plan.preset.replica_spec_groups ], diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md index 9867a0b790..34f9775396 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md @@ -33,10 +33,26 @@ Hardware behavior: - Do not blindly select the cheapest offer. - Prefer hardware likely to run the serving image reliably: enough VRAM, enough disk, common CUDA-capable NVIDIA GPUs when using CUDA images, and offers without obvious provisioning instability. +- Derive scheduling requirements from the model and serving method before looking at + concrete offers. Preview offers are placement evidence, not the target hardware spec. + Do not copy a preview offer's GPU name, region, instance type, CPU, memory, or disk into + the service YAML unless the model/framework actually requires it or you are intentionally + avoiding a proven failed class of hardware. +- Keep service `resources` as broad as correctness allows: minimum GPU memory/count, + required CPU/memory/disk, tensor-parallel needs, and endpoint/profile constraints. Exact + hardware belongs in the verified run evidence after provisioning, not in the initial plan. +- After a backend no-capacity or supply-constraint failure, do not just retry the same + concrete backend/region/GPU combination. Change the hypothesis or scheduling constraints + so dstack can try materially different viable offers, or stop and report why no credible + alternative remains. +- After the final service is running, re-read `dstack run get --json` and use + the actual latest job submission to identify the backend, region, price, instance type, + and resources that really provisioned. Do not infer final hardware from the service YAML, + the run name, or the first offer shown in a preview. - If a candidate stays in backend provisioning without logs/events progress after several polls, inspect run JSON/events and any available native backend or SSH/TCP evidence, then stop or fail with evidence rather than looping forever. - If a backend or dstack provisioning issue is found, create or reference a minimal non-endpoint reproduction in `endpoint-agent-backend-troubleshooting.md` when possible. Final report: -- On success, include the final service run id, final service run name, the exact final service YAML, recipe/source URLs, and a concise verification summary. +- On success, include the final service run id, final service run name, the exact final service YAML, recipe/source URLs, the actual provisioned hardware from run JSON, and a concise verification summary. - On failure, include the failure summary and enough evidence for the next iteration to improve the harness. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md b/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md index b39628e252..c6bf4627d5 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md @@ -20,6 +20,15 @@ Service essentials: - Common starting points are `vllm serve ` and `python -m sglang.launch_server --model-path --host 0.0.0.0 --port 8000`, but verify current docs and model-specific notes before trusting these defaults. - Pass endpoint environment variables by name, for example `env: [HF_TOKEN]`; never write secret values into YAML or logs. - Choose `resources` from evidence: model size, framework requirements, GPU memory, disk needed for weights/cache, and current dstack offers/fleets. +- Do not turn the first matching offer into the service requirements. Use offer previews to + verify that the derived requirements are provisionable. Keep `resources` as ranges or + minimums whenever that is enough for correctness. +- Do not pin `gpu.name`, `instance_types`, regions, CPU, memory, or disk from a preview + offer unless there is a model/framework requirement or a documented provisioning reason. + The actual instance that succeeds is recorded after provisioning, not guessed before it. - For OpenAI-compatible verification, use `service.model.base_url` from `dstack run get --json` and POST to `/chat/completions` with the requested model. If the base URL is relative, prefix it with the default project URL from `~/.dstack/config.yml`. +- After verification, read `dstack run get --json` again and report actual + provisioned hardware from the latest running job. If dstack fell back from one preview + offer to another, the final report must name the hardware that actually ran. Use dstack plans as real constraints. If the endpoint supplied `max_price`, `spot_policy`, backend, region, instance type, fleet, or reuse constraints, every offer lookup, preview, and submitted experiment must honor them. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md b/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md index 1c706f7c04..527cf51ae5 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md @@ -19,6 +19,7 @@ Recipe selection rubric: - Prefer an official framework recipe for the exact model. If none exists, use the model family recipe plus the model card. - Check whether the model needs `trust_remote_code`, special tokenizer/chat template handling, quantization, tensor parallelism, or specific framework versions. -- Estimate VRAM and disk before looking at offers. For a tiny model, avoid overbuying unless the cheaper path is unstable. For larger models, do not guess; use model size, precision, quantization, KV-cache, and tensor-parallel evidence. +- Estimate VRAM, GPU count, disk, and any CPU/memory needs before looking at offers. For a tiny model, avoid overbuying unless the cheaper path is unstable. For larger models, do not guess; use model size, precision, quantization, KV-cache, and tensor-parallel evidence. +- Treat concrete offers as availability evidence, not as requirements. A selected offer may fail or dstack may provision a different matching offer; final hardware evidence must come from the verified run. - When evidence is uncertain, use a bounded experiment instead of guessing. - Record recipe/source URLs in the final report so a learned preset has provenance. diff --git a/src/dstack/_internal/server/services/endpoints/preset_building.py b/src/dstack/_internal/server/services/endpoints/preset_building.py index 6c3860590a..24fbff8040 100644 --- a/src/dstack/_internal/server/services/endpoints/preset_building.py +++ b/src/dstack/_internal/server/services/endpoints/preset_building.py @@ -38,7 +38,8 @@ def build_endpoint_preset_from_run(name: str, run_model: RunModel) -> EndpointPr replica_spec_groups.append( EndpointPresetReplicaSpecGroup( name=group_name, - replica_specs=[_get_job_resources_spec(job) for job in group_jobs], + resources=replica_group.resources, + tested_resources=[_get_job_resources_spec(job) for job in group_jobs], ) ) @@ -68,9 +69,9 @@ def _get_current_registered_replica_jobs_by_group( def _get_job_resources_spec(job_model: JobModel) -> ResourcesSpec: offer = _get_job_offer(job_model) - if offer is not None: - return _resources_spec_from_instance_resources(offer.instance.resources) - return jobs_services.get_job_spec(job_model).requirements.resources + if offer is None: + raise ValueError("endpoint preset cannot be built without actual instance resources") + return _resources_spec_from_instance_resources(offer.instance.resources) def _get_job_offer(job_model: JobModel) -> InstanceOfferWithAvailability | None: @@ -107,4 +108,6 @@ def _resources_spec_from_instance_resources(resources: Resources) -> ResourcesSp } if first_gpu.vendor is not None: data["gpu"]["vendor"] = first_gpu.vendor.value + else: + data["gpu"] = 0 return ResourcesSpec.parse_obj(data) diff --git a/src/dstack/_internal/server/services/endpoints/presets.py b/src/dstack/_internal/server/services/endpoints/presets.py index bef7591896..70ddc0cd31 100644 --- a/src/dstack/_internal/server/services/endpoints/presets.py +++ b/src/dstack/_internal/server/services/endpoints/presets.py @@ -8,7 +8,7 @@ from typing import Any, Optional, Sequence import yaml -from pydantic import ValidationError +from pydantic import ValidationError, parse_obj_as from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import ( @@ -20,21 +20,23 @@ ) from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.profiles import ProfileParams -from dstack._internal.core.models.resources import ResourcesSpec +from dstack._internal.core.models.resources import CPUSpec, ResourcesSpec from dstack._internal.server import settings from dstack._internal.utils.common import run_async from dstack._internal.utils.logging import get_logger -logger = get_logger(__name__) - _SECRET_ENV_PATTERN = re.compile(r"(token|key|secret|password)", re.IGNORECASE) +logger = get_logger(__name__) class EndpointPresetReplicaSpecGroup(CoreModel): """Ordered to match `ServiceConfiguration.replica_groups`; "0" is the implicit group.""" name: str - replica_specs: list[ResourcesSpec] + resources: ResourcesSpec + """Per-replica scheduling requirements used when applying the preset.""" + tested_resources: list[ResourcesSpec] + """Exact resources of the replicas that were running when the preset was verified.""" class EndpointPreset(CoreModel): @@ -135,6 +137,7 @@ def _load_preset(self, path: Path) -> Optional[EndpointPreset]: return None def _save_preset(self, preset: EndpointPreset, comments: Sequence[str]) -> EndpointPreset: + _validate_preset_before_save(preset) self._presets_dir.mkdir(parents=True, exist_ok=True) data = _preset_to_data(preset) content = _format_preset_comments(comments) + yaml.safe_dump(data, sort_keys=False) @@ -184,6 +187,16 @@ def _get_preset_name_from_path(path: Path) -> str: return name +def _validate_preset_before_save(preset: EndpointPreset) -> None: + expected_names = [ + group.name or DEFAULT_REPLICA_GROUP_NAME for group in preset.configuration.replica_groups + ] + _validate_replica_spec_groups( + replica_spec_groups=preset.replica_spec_groups, + expected_names=expected_names, + ) + + def _get_service_data(data: dict[str, Any]) -> dict[str, Any]: service_data = data.get("service") if not isinstance(service_data, dict): @@ -244,9 +257,9 @@ def _service_configuration_to_preset_data( def _replica_spec_group_to_data(group: EndpointPresetReplicaSpecGroup) -> dict[str, Any]: return { "name": group.name, - "replica_specs": [ - json.loads(replica_spec.json(exclude_none=True)) - for replica_spec in group.replica_specs + "resources": json.loads(group.resources.json(exclude_none=True)), + "tested_resources": [ + json.loads(resources.json(exclude_none=True)) for resources in group.tested_resources ], } @@ -304,6 +317,9 @@ def _get_preset_replica_spec_groups( expected_names = _get_expected_replica_group_names(service_data) if expected_names == [DEFAULT_REPLICA_GROUP_NAME] and "name" not in raw_replica_spec_groups[0]: raw_replica_spec_groups[0]["name"] = DEFAULT_REPLICA_GROUP_NAME + raw_replica_spec_groups = [ + _normalize_replica_spec_group(group) for group in raw_replica_spec_groups + ] replica_spec_groups = [ EndpointPresetReplicaSpecGroup.parse_obj(group) for group in raw_replica_spec_groups ] @@ -317,6 +333,28 @@ def _get_preset_replica_spec_groups( return replica_spec_groups +def _normalize_replica_spec_group(group: dict[str, Any]) -> dict[str, Any]: + if "resources" in group or "tested_resources" in group: + if "resources" not in group or "tested_resources" not in group: + raise ValueError( + "preset replica_spec_groups must specify resources and tested_resources" + ) + if "replica_specs" in group: + raise ValueError( + "preset replica_spec_groups must not mix replica_specs with resources" + ) + return group + + replica_specs = group.get("replica_specs") + if not isinstance(replica_specs, list) or not replica_specs: + raise ValueError("preset replica_spec_groups must specify non-empty tested_resources") + group = dict(group) + group["resources"] = replica_specs[0] + group["tested_resources"] = replica_specs + group.pop("replica_specs", None) + return group + + def _get_expected_replica_group_names(service_data: dict[str, Any]) -> list[str]: replicas = service_data.get("replicas") if not isinstance(replicas, list): @@ -346,11 +384,44 @@ def _validate_replica_spec_groups( + ", ".join(expected_names) ) for group in replica_spec_groups: - if not group.replica_specs: - raise ValueError("preset replica_spec_groups must specify non-empty replica_specs") - first_resources = group.replica_specs[0].dict() - if any(resources.dict() != first_resources for resources in group.replica_specs): - raise ValueError("preset replica_specs within one group must have the same resources") + if not group.tested_resources: + raise ValueError("preset replica_spec_groups must specify non-empty tested_resources") + for resources in group.tested_resources: + _validate_replica_resources_are_exact(resources) + + +def _validate_replica_resources_are_exact(resources: ResourcesSpec) -> None: + cpu = parse_obj_as(CPUSpec, resources.cpu) + if not _is_exact_range(cpu.count): + _raise_loose_replica_resources() + if not _is_exact_range(resources.memory): + _raise_loose_replica_resources() + if resources.disk is None or not _is_exact_range(resources.disk.size): + _raise_loose_replica_resources() + if resources.gpu is None or not _is_exact_range(resources.gpu.count): + _raise_loose_replica_resources() + gpu_count = resources.gpu.count.min + if gpu_count == 0: + return + if resources.gpu.name is None or len(resources.gpu.name) != 1: + _raise_loose_replica_resources() + if resources.gpu.memory is None or not _is_exact_range(resources.gpu.memory): + _raise_loose_replica_resources() + if resources.gpu.compute_capability is not None: + _raise_loose_replica_resources() + + +def _raise_loose_replica_resources() -> None: + raise ValueError("preset tested_resources must use exact replica resources") + + +def _is_exact_range(value) -> bool: + return ( + value is not None + and value.min is not None + and value.max is not None + and value.min == value.max + ) def _apply_replica_spec_group_resources( @@ -359,10 +430,8 @@ def _apply_replica_spec_group_resources( ) -> None: replicas = service_data.get("replicas") if not isinstance(replicas, list): - service_data["resources"] = replica_spec_groups[0].replica_specs[0].dict() + service_data["resources"] = replica_spec_groups[0].resources.dict() return - resources_by_group = { - group.name: group.replica_specs[0].dict() for group in replica_spec_groups - } + resources_by_group = {group.name: group.resources.dict() for group in replica_spec_groups} for group in replicas: group["resources"] = resources_by_group[group["name"]] diff --git a/src/tests/_internal/cli/utils/test_preset.py b/src/tests/_internal/cli/utils/test_preset.py index 841de7217c..b2c0b1baa4 100644 --- a/src/tests/_internal/cli/utils/test_preset.py +++ b/src/tests/_internal/cli/utils/test_preset.py @@ -12,7 +12,18 @@ def test_shows_single_group_preset(self): model="Qwen/Qwen3-0.6B", replica_spec_groups=[ EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "0", "replica_specs": [{"gpu": "16GB", "disk": "60GB"}]} + { + "name": "0", + "resources": {"gpu": "16GB"}, + "tested_resources": [ + { + "cpu": 9, + "memory": "50GB", + "disk": "60GB", + "gpu": {"name": "A40", "memory": "48GB", "count": 1}, + } + ], + } ) ], ) @@ -21,8 +32,47 @@ def test_shows_single_group_preset(self): assert table.columns[0]._cells == ["qwen-qwen3-0-6b"] assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B"] - assert "gpu=16GB" in table.columns[2]._cells[0] - assert "disk=60GB" in table.columns[2]._cells[0] + assert table.columns[2]._cells == [ + "cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1", + ] + + def test_shows_individual_replicas_without_group_for_implicit_group(self): + preset = EndpointPreset( + name="qwen", + model="Qwen/Qwen3-0.6B", + replica_spec_groups=[ + EndpointPresetReplicaSpecGroup.parse_obj( + { + "name": "0", + "resources": {"gpu": "16GB"}, + "tested_resources": [ + { + "cpu": 9, + "memory": "50GB", + "disk": "60GB", + "gpu": {"name": "A40", "memory": "48GB", "count": 1}, + }, + { + "cpu": 9, + "memory": "50GB", + "disk": "60GB", + "gpu": {"name": "A40", "memory": "48GB", "count": 1}, + }, + ], + } + ) + ], + ) + + table = get_endpoint_presets_table([preset]) + + assert table.columns[0]._cells == ["qwen", " replica=0", " replica=1"] + assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B", "", ""] + assert table.columns[2]._cells == [ + "", + "cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1", + "cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1", + ] def test_shows_grouped_replica_specs(self): preset = EndpointPreset( @@ -30,18 +80,49 @@ def test_shows_grouped_replica_specs(self): model="Qwen/Qwen3-30B-A3B", replica_spec_groups=[ EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "router", "replica_specs": [{"cpu": 4}]} + { + "name": "router", + "resources": {"cpu": 4}, + "tested_resources": [ + {"cpu": 8, "memory": "16GB", "disk": "100GB", "gpu": 0} + ], + } ), EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "worker", "replica_specs": [{"gpu": "24GB"}, {"gpu": "24GB"}]} + { + "name": "worker", + "resources": {"gpu": "24GB"}, + "tested_resources": [ + { + "cpu": 14, + "memory": "64GB", + "disk": "200GB", + "gpu": {"name": "L4", "memory": "24GB", "count": 1}, + }, + { + "cpu": 14, + "memory": "64GB", + "disk": "200GB", + "gpu": {"name": "L4", "memory": "24GB", "count": 1}, + }, + ], + } ), ], ) table = get_endpoint_presets_table([preset]) - resources = table.columns[2]._cells[0] - assert "router:" in resources - assert "worker:" in resources - assert "2x" in resources - assert "gpu=24GB" in resources + assert table.columns[0]._cells == [ + "qwen-pd", + " group=router replica=0", + " group=worker replica=0", + " replica=1", + ] + assert table.columns[1]._cells == ["Qwen/Qwen3-30B-A3B", "", "", ""] + assert table.columns[2]._cells == [ + "", + "cpu=8 mem=16GB disk=100GB", + "cpu=14 mem=64GB disk=200GB gpu=L4:24GB:1", + "cpu=14 mem=64GB disk=200GB gpu=L4:24GB:1", + ] diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index 1169876024..5520e0b1f7 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -8,9 +8,19 @@ from sqlalchemy.orm import joinedload from dstack._internal.core.errors import ServerClientError +from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus from dstack._internal.core.models.envs import Env +from dstack._internal.core.models.instances import ( + Disk, + Gpu, + InstanceAvailability, + InstanceOfferWithAvailability, + InstanceRuntime, + InstanceType, + Resources, +) from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus, ServiceSpec from dstack._internal.server.background.pipeline_tasks.endpoints import ( EndpointPipelineItem, @@ -29,6 +39,7 @@ create_repo, create_run, create_user, + get_job_runtime_data, list_events, ) @@ -164,11 +175,32 @@ async def _create_backing_service_run( run=run, status=job_status, registered=registered, + job_runtime_data=get_job_runtime_data(offer=_instance_offer()), ) await session.commit() return run +def _instance_offer() -> InstanceOfferWithAvailability: + return InstanceOfferWithAvailability( + backend=BackendType.AWS, + instance=InstanceType( + name="g5.xlarge", + resources=Resources( + cpus=4, + memory_mib=16 * 1024, + gpus=[Gpu(name="T4", memory_mib=16 * 1024)], + spot=False, + disk=Disk(size_mib=100 * 1024), + ), + ), + region="us-east-1", + price=1.0, + availability=InstanceAvailability.AVAILABLE, + instance_runtime=InstanceRuntime.SHIM, + ) + + async def _create_ready_backing_service_run_for_agent( endpoint_id: uuid.UUID, ) -> AgentProvisioningResult: diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index caa91d9f92..de0ee22701 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -481,7 +481,8 @@ async def test_lists_endpoint_presets( assert body[0]["name"] == "qwen" assert body[0]["model"] == "Qwen/Qwen3-0.6B" assert body[0]["replica_spec_groups"][0]["name"] == "0" - assert body[0]["replica_spec_groups"][0]["replica_specs"][0]["gpu"] is not None + assert body[0]["replica_spec_groups"][0]["resources"]["gpu"] is not None + assert body[0]["replica_spec_groups"][0]["tested_resources"][0]["gpu"] is not None @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -567,7 +568,18 @@ def _endpoint_preset_plan() -> EndpointPresetPlan: model="Qwen/Qwen3-0.6B", replica_spec_groups=[ EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "0", "replica_specs": [{"gpu": "16GB"}]} + { + "name": "0", + "resources": {"gpu": "16GB"}, + "tested_resources": [ + { + "cpu": 4, + "memory": "16GB", + "disk": "100GB", + "gpu": {"name": "T4", "memory": "16GB", "count": 1}, + } + ], + } ) ], configuration=service_configuration, diff --git a/src/tests/_internal/server/services/test_endpoint_presets.py b/src/tests/_internal/server/services/test_endpoint_presets.py index d048148a08..57e8980b8b 100644 --- a/src/tests/_internal/server/services/test_endpoint_presets.py +++ b/src/tests/_internal/server/services/test_endpoint_presets.py @@ -57,8 +57,16 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 port: 8000 replica_spec_groups: - - replica_specs: - - gpu: 16GB + - resources: + gpu: 16GB + tested_resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """ ) @@ -69,7 +77,8 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): assert presets[0].model == "Qwen/Qwen3-4B" assert len(presets[0].replica_spec_groups) == 1 assert presets[0].replica_spec_groups[0].name == "0" - assert presets[0].replica_spec_groups[0].replica_specs[0].gpu is not None + assert presets[0].replica_spec_groups[0].resources.gpu is not None + assert presets[0].replica_spec_groups[0].tested_resources[0].gpu is not None assert presets[0].configuration.resources.gpu is not None @pytest.mark.asyncio @@ -94,12 +103,31 @@ async def test_lists_replica_group_presets_in_group_order(self, tmp_path): - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 replica_spec_groups: - name: router - replica_specs: - - gpu: 16GB + resources: + cpu: 4 + tested_resources: + - cpu: 8 + memory: 16GB + disk: 100GB + gpu: 0 - name: worker - replica_specs: - - gpu: 24GB - - gpu: 24GB + resources: + gpu: 24GB + tested_resources: + - cpu: 14 + memory: 64GB + disk: 200GB + gpu: + name: L4 + memory: 24GB + count: 1 + - cpu: 14 + memory: 64GB + disk: 200GB + gpu: + name: L4 + memory: 24GB + count: 1 """ ) @@ -109,7 +137,10 @@ async def test_lists_replica_group_presets_in_group_order(self, tmp_path): assert [group.name for group in presets[0].replica_spec_groups] == ["router", "worker"] replica_groups = presets[0].configuration.replica_groups assert [group.name for group in replica_groups] == ["router", "worker"] - assert all(group.resources.gpu is not None for group in replica_groups) + assert replica_groups[0].resources.gpu is not None + assert replica_groups[0].resources.gpu.count.min == 0 + assert replica_groups[0].resources.cpu.count.min == 4 + assert replica_groups[1].resources.gpu is not None @pytest.mark.asyncio async def test_saves_preset_without_overwriting(self, tmp_path): @@ -136,7 +167,7 @@ async def test_saves_preset_without_overwriting(self, tmp_path): assert "SECRET_VALUE" not in text assert "preset-service-name" not in text assert "creation_policy" not in text - assert "resources:" not in text + assert "tested_resources:" in text assert "HF_HOME=/root/.cache/huggingface" in text assert "HF_TOKEN" in text @@ -176,18 +207,44 @@ async def test_delete_missing_preset_raises(self, tmp_path): await service.delete_preset("missing") @pytest.mark.asyncio - async def test_skips_invalid_presets(self, tmp_path): - (tmp_path / "task.yml").write_text("type: task\ncommands:\n - echo nope\n") - (tmp_path / "missing-model.yml").write_text( - """\ -type: service -commands: - - python -m http.server 8000 -port: 8000 -""" - ) - (tmp_path / "mixed-resources.yml").write_text( - """\ + async def test_ignores_non_yaml_files(self, tmp_path): + (tmp_path / "notes.txt").write_text("ignored") + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + + assert presets == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("filename", "content", "message"), + [ + ( + "task.yml", + "type: task\ncommands:\n - echo nope\n", + "preset must be an endpoint preset", + ), + ( + "missing-model.yml", + """\ +type: endpoint-preset +service: + commands: + - python -m http.server 8000 + port: 8000 +replica_spec_groups: + - resources: + gpu: 16GB + tested_resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: 0 +""", + "preset must specify a model", + ), + ( + "mixed-resources.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: @@ -197,12 +254,22 @@ async def test_skips_invalid_presets(self, tmp_path): resources: gpu: 24GB replica_spec_groups: - - replica_specs: - - gpu: 16GB -""" - ) - (tmp_path / "service-profile.yml").write_text( - """\ + - resources: + gpu: 16GB + tested_resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""", + "preset service object must not specify resources", + ), + ( + "service-profile.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: @@ -211,12 +278,22 @@ async def test_skips_invalid_presets(self, tmp_path): port: 8000 creation_policy: reuse replica_spec_groups: - - replica_specs: - - gpu: 16GB -""" - ) - (tmp_path / "group-resources.yml").write_text( - """\ + - resources: + gpu: 16GB + tested_resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""", + "preset service object must not specify profile fields", + ), + ( + "group-resources.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: @@ -230,11 +307,19 @@ async def test_skips_invalid_presets(self, tmp_path): replica_spec_groups: - name: worker replica_specs: - - gpu: 16GB -""" - ) - (tmp_path / "missing-replica-spec-groups.yml").write_text( - """\ + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""", + "preset service replica groups must not specify resources", + ), + ( + "missing-replica-spec-groups.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: @@ -243,10 +328,27 @@ async def test_skips_invalid_presets(self, tmp_path): - name: worker count: 1 image: vllm/vllm-openai:latest -""" - ) - (tmp_path / "mismatched-replica-spec-groups.yml").write_text( - """\ +""", + "preset must specify non-empty replica_spec_groups", + ), + ( + "loose-resources.yml", + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +service: + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 +replica_spec_groups: + - replica_specs: + - gpu: 16GB +""", + "preset tested_resources must use exact replica resources", + ), + ( + "mismatched-replica-spec-groups.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: @@ -258,28 +360,37 @@ async def test_skips_invalid_presets(self, tmp_path): replica_spec_groups: - name: other replica_specs: - - gpu: 16GB -""" - ) - (tmp_path / "heterogeneous-group.yml").write_text( - """\ + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""", + "preset replica_spec_groups must match replica group order", + ), + ( + "missing-tested-resources.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: port: 8000 replicas: - name: worker - count: 2 + count: 1 image: vllm/vllm-openai:latest replica_spec_groups: - name: worker - replica_specs: - - gpu: 16GB - - gpu: 24GB -""" - ) - (tmp_path / "bad-replica-group-shape.yml").write_text( - """\ + resources: + gpu: 16GB +""", + "preset replica_spec_groups must specify resources and tested_resources", + ), + ( + "bad-replica-group-shape.yml", + """\ type: endpoint-preset model: Qwen/Qwen3-4B service: @@ -289,15 +400,29 @@ async def test_skips_invalid_presets(self, tmp_path): replica_spec_groups: - name: worker replica_specs: - - gpu: 16GB -""" - ) - (tmp_path / "not-yaml.yml").write_text(":\n") - (tmp_path / "notes.txt").write_text("ignored") + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""", + "preset service replica groups must be objects", + ), + ("not-yaml.yml", ":\n", "while parsing a block mapping"), + ], + ) + async def test_invalid_preset_is_skipped_and_logged( + self, tmp_path, caplog, filename, content, message + ): + (tmp_path / filename).write_text(content) presets = await LocalDirEndpointPresetService(tmp_path).list_presets() assert presets == [] + assert filename in caplog.text + assert message in caplog.text class TestBuildEndpointPresetFromRun: @@ -348,9 +473,49 @@ async def test_builds_implicit_replica_group_from_running_service( assert saved.name == "qwen-learned" assert saved.model == "Qwen/Qwen3-4B" assert [group.name for group in saved.replica_spec_groups] == ["0"] - resources = saved.replica_spec_groups[0].replica_specs[0].pretty_format() - assert "cpu=14" in resources - assert "gpu=L4:24GB:1" in resources + assert "gpu=16GB" in saved.replica_spec_groups[0].resources.pretty_format() + tested_resources = saved.replica_spec_groups[0].tested_resources[0].pretty_format() + assert "cpu=14" in tested_resources + assert "gpu=L4:24GB:1" in tested_resources + + @pytest.mark.asyncio + async def test_requires_actual_instance_resources(self, session: AsyncSession): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + run_name="qwen-endpoint", + status=RunStatus.RUNNING, + run_spec=RunSpec( + run_name="qwen-endpoint", + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "qwen-endpoint", + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-4B", + "resources": {"gpu": "16GB"}, + } + ), + ), + ) + await create_job( + session=session, + run=run, + status=JobStatus.RUNNING, + registered=True, + ) + await session.refresh(run, attribute_names=["jobs"]) + + with pytest.raises(ValueError, match="actual instance resources"): + build_endpoint_preset_from_run("qwen-learned", run) @pytest.mark.asyncio async def test_builds_replica_groups_in_service_order(self, session: AsyncSession, tmp_path): @@ -429,8 +594,10 @@ async def test_builds_replica_groups_in_service_order(self, session: AsyncSessio saved = await LocalDirEndpointPresetService(tmp_path).save_preset(preset) assert [group.name for group in saved.replica_spec_groups] == ["router", "worker"] - assert len(saved.replica_spec_groups[0].replica_specs) == 1 - assert len(saved.replica_spec_groups[1].replica_specs) == 2 + assert "cpu=4" in saved.replica_spec_groups[0].resources.pretty_format() + assert "gpu=24GB" in saved.replica_spec_groups[1].resources.pretty_format() + assert len(saved.replica_spec_groups[0].tested_resources) == 1 + assert len(saved.replica_spec_groups[1].tested_resources) == 2 assert [group.name for group in saved.configuration.replica_groups] == [ "router", "worker", @@ -451,9 +618,7 @@ def test_endpoint_name_env_and_profile_are_applied(self): name="qwen", model="Qwen/Qwen3-4B", replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "0", "replica_specs": [{"gpu": "16GB"}]} - ) + EndpointPresetReplicaSpecGroup.parse_obj(_t4_replica_spec_group()) ], configuration=ServiceConfiguration.parse_obj( { @@ -497,9 +662,7 @@ def test_builds_repo_less_run_spec(self): name="qwen", model="Qwen/Qwen3-4B", replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "0", "replica_specs": [{"gpu": "16GB"}]} - ) + EndpointPresetReplicaSpecGroup.parse_obj(_t4_replica_spec_group()) ], configuration=ServiceConfiguration.parse_obj( { @@ -647,8 +810,16 @@ async def test_skips_preset_with_unresolved_env_sentinel( - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 port: 8000 replica_spec_groups: - - replica_specs: - - gpu: 16GB + - resources: + gpu: 16GB + tested_resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """ ) user = await create_user( @@ -722,8 +893,16 @@ def _write_qwen_preset(tmp_path): - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 port: 8000 replica_spec_groups: - - replica_specs: - - gpu: 16GB + - resources: + gpu: 16GB + tested_resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """ ) @@ -765,11 +944,7 @@ def _qwen_preset(name: str) -> EndpointPreset: return EndpointPreset( name=name, model="Qwen/Qwen3-4B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( - {"name": "0", "replica_specs": [{"gpu": "16GB"}]} - ) - ], + replica_spec_groups=[EndpointPresetReplicaSpecGroup.parse_obj(_t4_replica_spec_group())], configuration=ServiceConfiguration.parse_obj( { "type": "service", @@ -790,6 +965,21 @@ def _qwen_preset(name: str) -> EndpointPreset: ) +def _t4_replica_spec_group() -> dict: + return { + "name": "0", + "resources": {"gpu": "16GB"}, + "tested_resources": [ + { + "cpu": 4, + "memory": "16GB", + "disk": "100GB", + "gpu": {"name": "T4", "memory": "16GB", "count": 1}, + } + ], + } + + def _run_plan_with_offer(available: bool): availability = Mock() availability.is_available.return_value = available From dccab947cf4f7bb8ce0606934ebf22c3cb8d30f3 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 22:02:23 +0200 Subject: [PATCH 10/21] Refine endpoint agent status and logs --- endpoint-agent-checkpoints.md | 44 ++++ .../background/pipeline_tasks/endpoints.py | 74 +++++- .../server/services/endpoints/agent/claude.py | 241 ++++++++---------- .../agent/resources/deployment_harness.md | 1 + .../pipeline_tasks/test_endpoints.py | 66 +++-- .../services/endpoints/test_claude_agent.py | 93 ++++++- 6 files changed, 360 insertions(+), 159 deletions(-) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index 6f2cb47df1..2dcc8c9a60 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -405,3 +405,47 @@ These are outside the repo and are not part of the checkpoint commit: `dstack logs ENDPOINT`. Endpoint logs should contain major realtime events only: research/plan summary, candidate submitted, provisioning state, service startup, verification success/failure, preset save, and cleanup. + +### Post-Checkpoint Patch: Agent Status And Logs + +Implemented immediately after tag `endpoint-agent/qwen-runpod-v1-endpoint-dev-running`. +The tag remains the recovery point for the known-good live RunPod smoke; these edits +are the next local branch commit. + +Status behavior: + +- Agent-created endpoints no longer expose an intermediate `provisioning` state after + the agent returns a verified service run. +- If the reported service is not yet fully visible as a ready dstack service, the + endpoint stays `agenting` with `service_run_id` linked. +- Once the linked service is ready, the worker saves the learned preset and moves the + endpoint directly from `agenting` to `running`. +- Preset-based endpoint creation still uses `provisioning`. + +Endpoint log behavior: + +- Claude stream/tool output is no longer copied to endpoint logs. +- Full agent trace and command output remain in workspace artifacts: + `trace.jsonl` when debug is enabled, plus `commands.jsonl` and `command-output/`. +- The agent now has a user-facing progress protocol: append concise JSON objects to + `progress.jsonl`, for example + `{"phase":"submit","message":"Submitted service candidate"}`. +- The server tails `progress.jsonl` and writes only those major events to the configured + log service for `dstack logs ENDPOINT`. +- Endpoint logs still include concise start/finish markers for the provisioning agent. + +Verification on 2026-07-04: + +```bash +uv run pytest src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py +uv run ruff check src/dstack/_internal/server/background/pipeline_tasks/endpoints.py src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +uv run ruff format --check src/dstack/_internal/server/background/pipeline_tasks/endpoints.py src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +``` + +Observed result: + +- focused endpoint/agent pytest: `51 passed, 36 skipped` +- broader endpoint/log pytest: `104 passed, 11 skipped` +- ruff: `All checks passed!` +- format check: `4 files already formatted` diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py index cf760f89ae..701a2a170f 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -511,13 +511,40 @@ async def _process_agenting_endpoint( pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: if endpoint_model.service_run is not None: - return _ProcessResult(update_map={"status": EndpointStatus.PROVISIONING}) + return await _process_agent_verified_endpoint(endpoint_model) return await _provision_endpoint_with_agent( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) +async def _process_agent_verified_endpoint(endpoint_model: EndpointModel) -> _ProcessResult: + run_model = endpoint_model.service_run + if run_model is None: + return _ProcessResult() + readiness = _get_service_run_readiness(run_model, endpoint_name=endpoint_model.name) + if readiness.failed_message is not None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": readiness.failed_message, + } + ) + if readiness.model_base_url is None or readiness.model_name is None: + return _ProcessResult(update_map={"status": EndpointStatus.AGENTING}) + await _save_agent_endpoint_preset( + endpoint_model=endpoint_model, + run_model=run_model, + model_name=readiness.model_name, + ) + return _ProcessResult( + update_map={ + "status": EndpointStatus.RUNNING, + "status_message": None, + } + ) + + async def _provision_endpoint_with_agent( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, @@ -643,9 +670,30 @@ async def _provision_endpoint_with_agent( "status_message": e.msg, } ) + readiness = _get_service_run_readiness(run_model, endpoint_name=endpoint_model.name) + if readiness.failed_message is not None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": readiness.failed_message, + } + ) + if readiness.model_base_url is None or readiness.model_name is None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.AGENTING, + "status_message": None, + "service_run_id": run_model.id, + } + ) + await _save_agent_endpoint_preset( + endpoint_model=endpoint_model, + run_model=run_model, + model_name=readiness.model_name, + ) return _ProcessResult( update_map={ - "status": EndpointStatus.PROVISIONING, + "status": EndpointStatus.RUNNING, "status_message": None, "service_run_id": run_model.id, } @@ -695,6 +743,18 @@ async def _try_save_agent_endpoint_preset( run_model = endpoint_model.service_run if run_model is None: return + await _save_agent_endpoint_preset( + endpoint_model=endpoint_model, + run_model=run_model, + model_name=model_name, + ) + + +async def _save_agent_endpoint_preset( + endpoint_model: EndpointModel, + run_model: RunModel, + model_name: str, +) -> None: preset_name = f"{model_name}-{str(run_model.id)[:8]}" try: preset = build_endpoint_preset_from_run(name=preset_name, run_model=run_model) @@ -729,6 +789,14 @@ def _get_backing_service_readiness(endpoint_model: EndpointModel) -> _BackingSer run_model = endpoint_model.service_run if run_model is None: return _BackingServiceReadiness(failed_message="Backing service run is missing") + return _get_service_run_readiness(run_model, endpoint_name=endpoint_model.name) + + +def _get_service_run_readiness( + run_model: RunModel, + *, + endpoint_name: str, +) -> _BackingServiceReadiness: if run_model.deleted: return _BackingServiceReadiness(failed_message="Backing service run was deleted") if run_model.status.is_finished(): @@ -744,7 +812,7 @@ def _get_backing_service_readiness(endpoint_model: EndpointModel) -> _BackingSer try: service_spec = ServiceSpec.__response__.parse_raw(run_model.service_spec) except ValidationError: - logger.warning("Endpoint %s backing service spec is invalid", endpoint_model.name) + logger.warning("Endpoint %s backing service spec is invalid", endpoint_name) return _BackingServiceReadiness(failed_message="Backing service spec is invalid") if service_spec.model is None: return _BackingServiceReadiness() diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index 290862d220..020c836335 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -2,6 +2,7 @@ import json import os import shutil +from contextlib import suppress from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -54,9 +55,9 @@ _REDACTION = "[redacted]" _MAX_CAPTURED_OUTPUT_CHARS = 20_000 _MAX_AGENT_LOG_MESSAGE_CHARS = 4_000 -_MAX_TOOL_RESULT_LOG_PREVIEW_CHARS = 1_200 -_MAX_TOOL_RESULT_LOG_PREVIEW_LINES = 30 _AGENT_LOG_BATCH_SIZE = 1 +_AGENT_PROGRESS_LOG_NAME = "progress.jsonl" +_AGENT_PROGRESS_POLL_SECONDS = 1.0 _CLAUDE_AGENT_TOOLS = "Bash,Read,Write,Edit,WebFetch,WebSearch,StructuredOutput" @@ -185,6 +186,10 @@ def __init__( self.log_writer = log_writer self.artifacts = _AgentArtifactRecorder(self) + @property + def progress_path(self) -> Path: + return self.work_dir / _AGENT_PROGRESS_LOG_NAME + class _AgentLogWriter: def __init__( @@ -301,7 +306,12 @@ def __init__(self, workspace: _AgentWorkspace) -> None: def initialize(self) -> None: self._workspace.work_dir.mkdir(parents=True, exist_ok=True) self._update_agent_state(phase="starting") - for filename in ["sources.jsonl", "candidates.jsonl", "commands.jsonl"]: + for filename in [ + "sources.jsonl", + "candidates.jsonl", + "commands.jsonl", + _AGENT_PROGRESS_LOG_NAME, + ]: (self._workspace.work_dir / filename).touch(exist_ok=True) hardware_reasoning_path = self._workspace.work_dir / "hardware_reasoning.md" if not hardware_reasoning_path.exists(): @@ -610,22 +620,30 @@ async def _run_agent_in_subprocess( _format_agent_start_log(request), ) workspace.artifacts.mark_running() - proc = await asyncio.create_subprocess_exec( - *cmd, - cwd=workspace.work_dir, - env=request["env"], - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - assert proc.stdout is not None - assert proc.stderr is not None - stdout_task = asyncio.create_task(_read_agent_stdout(proc.stdout, workspace)) - stderr_task = asyncio.create_task(_read_agent_stderr(proc.stderr, workspace)) - stdout_output, stderr_output, returncode = await asyncio.gather( - stdout_task, - stderr_task, - proc.wait(), - ) + progress_tailer = _AgentProgressLogTailer(workspace) + progress_task = asyncio.create_task(progress_tailer.run()) + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=workspace.work_dir, + env=request["env"], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert proc.stdout is not None + assert proc.stderr is not None + stdout_task = asyncio.create_task(_read_agent_stdout(proc.stdout, workspace)) + stderr_task = asyncio.create_task(_read_agent_stderr(proc.stderr, workspace)) + stdout_output, stderr_output, returncode = await asyncio.gather( + stdout_task, + stderr_task, + proc.wait(), + ) + finally: + progress_task.cancel() + with suppress(asyncio.CancelledError): + await progress_task + await progress_tailer.flush(include_partial=True) process_output = _merge_process_outputs(stdout_output, stderr_output) _write_trace_record( workspace, @@ -634,7 +652,7 @@ async def _run_agent_in_subprocess( "returncode": returncode, }, ) - await _write_agent_log(workspace, f"Server agent process exited with code {returncode}") + await _write_agent_log(workspace, _format_agent_exit_log(returncode)) await _flush_agent_logs(workspace) if process_output.report_data is None: process_output.report_data = _load_final_report_artifact(workspace) @@ -730,14 +748,78 @@ async def _read_agent_stream( except json.JSONDecodeError: message = {"type": "raw-output", "stream": stream_name, "line": line.rstrip("\r\n")} _write_trace_record(workspace, message) - await _write_agent_log_if_present(workspace, message) continue if stream_name != "stdout": message.setdefault("stream", stream_name) _write_trace_record(workspace, message) workspace.artifacts.record_stream_message(message) _update_agent_process_output(output, message) - await _write_agent_log_if_present(workspace, message) + + +class _AgentProgressLogTailer: + def __init__(self, workspace: _AgentWorkspace) -> None: + self._workspace = workspace + self._offset = 0 + self._partial = "" + + async def run(self) -> None: + while True: + await self.flush() + await asyncio.sleep(_AGENT_PROGRESS_POLL_SECONDS) + + async def flush(self, *, include_partial: bool = False) -> None: + path = self._workspace.progress_path + if not path.exists(): + return + if path.stat().st_size < self._offset: + self._offset = 0 + self._partial = "" + with path.open("r", encoding="utf-8", errors="replace") as f: + f.seek(self._offset) + text = f.read() + self._offset = f.tell() + if not text and not (include_partial and self._partial): + return + self._partial += text + lines = self._partial.splitlines(keepends=True) + self._partial = "" + for line in lines: + if not include_partial and not line.endswith(("\n", "\r")): + self._partial = line + continue + await self._write_progress_line(line.rstrip("\r\n")) + + async def _write_progress_line(self, line: str) -> None: + if not line.strip(): + return + try: + parsed = json.loads(line) + except json.JSONDecodeError: + _write_trace_record( + self._workspace, + {"type": "agent-progress-invalid", "line": line}, + ) + return + message = _format_agent_progress_log_message(parsed) + if message is None: + _write_trace_record( + self._workspace, + {"type": "agent-progress-ignored", "record": parsed}, + ) + return + await _write_agent_log(self._workspace, message) + + +def _format_agent_progress_log_message(record: Any) -> Optional[str]: + if not isinstance(record, dict): + return None + message = record.get("message") + if not isinstance(message, str) or not message.strip(): + return None + phase = record.get("phase") + if isinstance(phase, str) and phase.strip(): + return f"{phase.strip()}: {message.strip()}" + return message.strip() def _update_agent_process_output( @@ -775,15 +857,6 @@ def _merge_process_outputs(*outputs: _AgentProcessOutput) -> _AgentProcessOutput return merged -async def _write_agent_log_if_present( - workspace: _AgentWorkspace, - message: dict[str, Any], -) -> None: - log_message = _format_agent_log_message(message) - if log_message is not None: - await _write_agent_log(workspace, log_message) - - async def _write_agent_log(workspace: _AgentWorkspace, message: str) -> None: if workspace.log_writer is None: return @@ -799,112 +872,16 @@ async def _flush_agent_logs(workspace: _AgentWorkspace) -> None: def _format_agent_start_log(request: dict[str, Any]) -> str: options = request["options"] - parts = [f"Starting server agent ({options['model']})"] + parts = [f"Starting endpoint provisioning agent ({options['model']})"] if options["max_budget"] is not None: parts.append(f"max budget ${options['max_budget']}") - cwd = request.get("cwd") - if cwd: - parts.append(f"workspace {cwd}") return ", ".join(parts) -def _format_agent_log_message(message: dict[str, Any]) -> Optional[str]: - message_type = message.get("type") - if message_type == "system" and message.get("subtype") == "init": - model = message.get("model") - version = message.get("claude_code_version") - details = ", ".join( - str(v) for v in [model, f"Claude Code {version}" if version else None] if v - ) - return f"Server agent initialized ({details})" if details else "Server agent initialized" - if message_type == "assistant": - return _format_assistant_message_log(message.get("message")) - if message_type == "user": - return _format_user_message_log(message.get("message")) - if message_type == "result": - if message.get("is_error"): - result = message.get("result") - if isinstance(result, str) and result.strip(): - return f"Server agent failed: {_truncate_log_message(result.strip())}" - return "Server agent failed" - return "Server agent finished" - if message_type == "raw-output": - line = message.get("line") - if isinstance(line, str) and line.strip(): - prefix = "Agent stderr" if message.get("stream") == "stderr" else "Agent output" - return f"{prefix}: {line.strip()}" - return None - - -def _format_assistant_message_log(message: Any) -> Optional[str]: - if not isinstance(message, dict): - return None - lines = [] - for item in message.get("content", []): - if not isinstance(item, dict): - continue - if item.get("type") == "tool_use": - name = item.get("name") or "tool" - tool_input = item.get("input") - description = None - command = None - if isinstance(tool_input, dict): - description = tool_input.get("description") - command = tool_input.get("command") - if isinstance(description, str) and description.strip(): - lines.append(f"Agent tool: {name} ({description.strip()})") - else: - lines.append(f"Agent tool: {name}") - if isinstance(command, str) and command.strip(): - lines.append(f"$ {command.strip()}") - elif item.get("type") == "text" and isinstance(item.get("text"), str): - text = item["text"].strip() - if text: - lines.append(text) - if not lines: - return None - return "\n".join(lines) - - -def _format_user_message_log(message: Any) -> Optional[str]: - if not isinstance(message, dict): - return None - lines = [] - for item in message.get("content", []): - if not isinstance(item, dict) or item.get("type") != "tool_result": - continue - content = item.get("content") - if not isinstance(content, str) or not content.strip(): - continue - prefix = "Tool error" if item.get("is_error") else "Tool output" - lines.append(_format_tool_result_log(prefix, content)) - if not lines: - return None - return "\n\n".join(lines) - - -def _format_tool_result_log(prefix: str, content: str) -> str: - stripped = content.strip() - result_lines = stripped.splitlines() - if ( - len(stripped) <= _MAX_TOOL_RESULT_LOG_PREVIEW_CHARS - and len(result_lines) <= _MAX_TOOL_RESULT_LOG_PREVIEW_LINES - ): - return f"{prefix}:\n{stripped}" - - preview_lines = result_lines[:_MAX_TOOL_RESULT_LOG_PREVIEW_LINES] - preview = "\n".join(preview_lines) - if len(preview) > _MAX_TOOL_RESULT_LOG_PREVIEW_CHARS: - preview = preview[: _MAX_TOOL_RESULT_LOG_PREVIEW_CHARS - 15].rstrip() - if preview: - preview += "\n... truncated" - else: - preview = "... truncated" - return ( - f"{prefix} captured ({len(stripped)} chars, {len(result_lines)} lines). " - "Full output is stored in the endpoint agent workspace.\n" - f"{preview}" - ) +def _format_agent_exit_log(returncode: Optional[int]) -> str: + if returncode == 0: + return "Endpoint provisioning agent finished" + return f"Endpoint provisioning agent exited with code {returncode}" def _truncate_log_message(message: str) -> str: diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md index 34f9775396..98f236e359 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md @@ -18,6 +18,7 @@ Workspace artifact contract: - On startup or resume, inspect `agent_state.json`, `candidates.jsonl`, `verification.json`, and `final_report.json` before submitting anything new. If a previous candidate is already running and verifiable, reuse it instead of creating a duplicate run. - Keep `agent_state.json` current enough to show the phase. +- Append user-facing major progress events to `progress.jsonl` in real time. Each line must be a small JSON object such as `{"phase":"research","message":"Checking Qwen serving recipes and model requirements"}`. Use this only for milestones: research direction, chosen experiment path, candidate submitted, provisioning observation, verification result, cleanup, or terminal failure. Do not write command output, YAML, secrets, long tables, or raw traces here. - Add source evidence to `sources.jsonl`. - Keep `hardware_reasoning.md` reviewable before any paid run. - Record every spend-capable service, task, or dev-environment candidate in `candidates.jsonl`. diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index 5520e0b1f7..568c7968b7 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -771,17 +771,26 @@ async def test_agenting_does_not_fail_on_legacy_same_name_run_conflict( run_name=_get_agent_run_name(), ) agent_service = _FakeAgentService(result=_get_verified_agent_result(agent_run)) + preset_service = Mock() + preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) - with patch( - "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", - return_value=agent_service, + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ), ): await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) await session.refresh(legacy_run) await session.refresh(agent_run) - assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status == EndpointStatus.RUNNING assert endpoint_model.status_message is None assert endpoint_model.service_run_id == agent_run.id assert legacy_run.status == RunStatus.PROVISIONING @@ -789,7 +798,7 @@ async def test_agenting_does_not_fail_on_legacy_same_name_run_conflict( assert agent_run.deleted is False events = await list_events(session) assert len(events) == 1 - assert events[0].message == "Endpoint status changed AGENTING -> PROVISIONING" + assert events[0].message == "Endpoint status changed AGENTING -> RUNNING" async def test_fails_when_agent_reports_foreign_run( self, test_db, session: AsyncSession, worker: EndpointWorker @@ -866,16 +875,11 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) await worker.process(_endpoint_to_pipeline_item(endpoint_model)) - await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.PROVISIONING - assert endpoint_model.service_run_id is not None - await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) - - await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) assert endpoint_model.status == EndpointStatus.RUNNING assert endpoint_model.status_message is None + assert endpoint_model.service_run_id is not None submissions = await _get_endpoint_run_submissions( session=session, endpoint_model=endpoint_model, @@ -923,15 +927,24 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): ) agent_service = _FakeAgentService(side_effect=provision_endpoint) + preset_service = Mock() + preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) - with patch( - "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", - return_value=agent_service, + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ), ): await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.PROVISIONING + assert endpoint_model.status == EndpointStatus.RUNNING assert endpoint_model.service_run_id is not None submissions = await _get_endpoint_run_submissions( session=session, @@ -939,6 +952,29 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): ) assert len(submissions) == 1 assert submissions[0].run_id == endpoint_model.service_run_id + preset_service.save_preset.assert_awaited_once() + + async def test_agenting_linked_run_waits_without_showing_provisioning( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.AGENTING, + ) + endpoint_model.provisioning_method = "agent" + await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.PROVISIONING, + ) + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.AGENTING + assert endpoint_model.status_message is None + events = await list_events(session) + assert events == [] async def test_agent_reported_run_without_verification_report_fails_endpoint( self, test_db, session: AsyncSession, worker: EndpointWorker diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index a85d6b61af..c4a0d08356 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -155,6 +155,7 @@ async def runner(workspace, request): assert (work_dir / "sources.jsonl").exists() assert (work_dir / "candidates.jsonl").exists() assert (work_dir / "commands.jsonl").exists() + assert (work_dir / "progress.jsonl").exists() assert (work_dir / "hardware_reasoning.md").exists() final_report = json.loads((work_dir / "final_report.json").read_text()) assert final_report["success"] is True @@ -245,6 +246,10 @@ async def runner(workspace, request): assert "Prototypes may be dstack services, tasks, or dev environments" in prompt assert "final verified run reported to the server must be a dstack service" in prompt assert "do not attempt P/D disaggregation" in prompt + assert "progress.jsonl" in prompt + assert ( + "Do not write command output, YAML, secrets, long tables, or raw traces here" in prompt + ) @pytest.mark.asyncio async def test_uses_server_default_agent_budget_when_endpoint_does_not_set_one( @@ -435,7 +440,7 @@ async def test_reads_claude_stream_incrementally(self, tmp_path): assert "[redacted]" in trace @pytest.mark.asyncio - async def test_writes_compact_agent_logs_from_claude_stream(self, tmp_path): + async def test_records_commands_without_copying_claude_stream_to_endpoint_logs(self, tmp_path): class FakeLogWriter: def __init__(self): self.messages = [] @@ -503,10 +508,7 @@ async def flush(self): await _read_agent_stdout(reader, workspace) - assert log_writer.messages == [ - "Agent tool: Bash (List offers)\n$ dstack offer --gpu 1 --max-price 0.3", - "Tool output:\nNo offers [redacted]", - ] + assert log_writer.messages == [] command_records = [ json.loads(line) for line in (tmp_path / "work" / "commands.jsonl").read_text().splitlines() @@ -519,7 +521,7 @@ async def flush(self): assert output == "No offers [redacted]" @pytest.mark.asyncio - async def test_truncates_large_tool_outputs_in_endpoint_logs(self, tmp_path): + async def test_stores_large_tool_outputs_without_endpoint_log_preview(self, tmp_path): class FakeLogWriter: def __init__(self): self.messages = [] @@ -567,9 +569,7 @@ async def flush(self): await _read_agent_stdout(reader, workspace) - assert len(log_writer.messages) == 1 - assert "Full output is stored in the endpoint agent workspace" in log_writer.messages[0] - assert "offer 0199" not in log_writer.messages[0] + assert log_writer.messages == [] command_records = [ json.loads(line) for line in (tmp_path / "work" / "commands.jsonl").read_text().splitlines() @@ -577,6 +577,81 @@ async def flush(self): output = (tmp_path / "work" / command_records[0]["output_path"]).read_text() assert output == long_output + @pytest.mark.asyncio + async def test_subprocess_streams_agent_progress_jsonl_to_endpoint_logs( + self, + tmp_path, + monkeypatch, + ): + class FakeLogWriter: + def __init__(self): + self.messages = [] + + async def write(self, message): + self.messages.append(message) + + async def flush(self): + pass + + claude_path = tmp_path / "claude" + claude_path.write_text( + """#!/bin/sh +printf '%s\\n' '{"phase":"research","message":"Checking recipes with secret"}' >> progress.jsonl +printf '%s\\n' '{"phase":"submit","message":"Submitted service candidate"}' >> progress.jsonl +cat > final_report.json <<'JSON' +{ + "success": true, + "run_id": "6e578748-d597-4fde-a3a4-203587cad5a2", + "run_name": "qwen-agent-candidate", + "service_yaml": "type: service\\nname: qwen-agent-candidate\\n", + "verification_summary": "Verified chat completions." +} +JSON +printf '%s\\n' '{"type":"result","is_error":false,"result":"done"}' +""", + encoding="utf-8", + ) + claude_path.chmod(0o755) + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", str(claude_path)) + work_dir = tmp_path / "work" + work_dir.mkdir() + log_writer = FakeLogWriter() + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=work_dir, + trace_path=None, + env={}, + redacted_values=["secret"], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + max_agent_budget=None, + log_writer=log_writer, + ) + request = { + "prompt": "prompt", + "env": {}, + "cwd": str(work_dir), + "options": { + "allowed_tools": "Bash", + "disallowed_tools": "", + "model": "test-model", + "max_turns": 1, + "max_budget": None, + "json_schema": {}, + }, + } + + result = await _run_agent_in_subprocess(workspace, request) + + assert result.error is None + assert log_writer.messages == [ + "Starting endpoint provisioning agent (test-model)", + "research: Checking recipes with [redacted]", + "submit: Submitted service candidate", + "Endpoint provisioning agent finished", + ] + @pytest.mark.asyncio async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missing( self, From 785d5e6e1773c9ec2a211e0a063097b4bfe9b39c Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 22:05:38 +0200 Subject: [PATCH 11/21] Mention endpoint agent skill guidance --- .../server/services/endpoints/agent/resources/system_prompt.md | 2 ++ .../_internal/server/services/endpoints/test_claude_agent.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md index 8737d8aa7d..87cc52fa1f 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md @@ -4,6 +4,8 @@ You are the server-side endpoint deployment agent for dstack. Your job is to tur This is a real deployment investigation, not a YAML-generation task. Use the local workspace to keep notes, service/dev/task YAML files, command transcripts, backend observations, and evidence. Use the real `dstack` CLI and shell commands directly. Verify CLI flags with `dstack --help` when unsure. +Use the embedded dstack skill guidance as the source of truth for normal dstack CLI workflows. Use the embedded dstack-development skill guidance when a dev environment would reduce uncertainty around images, commands, model download, serving parameters, or hardware before submitting the final service. + Do not invent dstack flags or YAML properties. Do not call hidden server APIs. Do not wait for custom helper functions such as `find_model_recipes`, `submit_service`, or `get_run_logs`; use normal files, shell commands, web sources, and the `dstack` CLI. You may use network research through the available web tools and shell commands. Prefer primary, current sources and preserve URLs in the final report. Treat model cards, official serving docs, recipe indexes, and successful command/log evidence as stronger than generic snippets. diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index c4a0d08356..41e81b9a69 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -236,6 +236,8 @@ async def runner(workspace, request): assert result.error is None prompt = captured["request"]["prompt"] assert "This is a real deployment investigation" in prompt + assert "embedded dstack skill guidance" in prompt + assert "embedded dstack-development skill guidance" in prompt assert "Do not call hidden server APIs" in prompt assert "dstack run get --json" in prompt assert "https://docs.vllm.ai/" in prompt From a9e61388a4e8423f2f074d26c8f35c728e36f1de Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sat, 4 Jul 2026 22:07:42 +0200 Subject: [PATCH 12/21] Move endpoint smoke configs under test configs --- .../endpoint-dev-runpod-fleet.dstack.yml | 11 +++++++++++ .../endpoint-agent/qwen-endpoint-happy.dstack.yml | 0 .../endpoint-agent/qwen-endpoint-log-smoke.dstack.yml | 9 +++++++++ endpoint-agent-checkpoints.md | 7 ++++--- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 .test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml rename qwen-endpoint-happy.dstack.yml => .test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml (100%) create mode 100644 .test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml diff --git a/.test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml b/.test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml new file mode 100644 index 0000000000..31ab9d3e12 --- /dev/null +++ b/.test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml @@ -0,0 +1,11 @@ +type: fleet +name: endpoint-dev-runpod + +nodes: 0..1 +backends: [runpod] +spot_policy: on-demand +max_price: 0.5 +idle_duration: 5m + +resources: + gpu: 1 diff --git a/qwen-endpoint-happy.dstack.yml b/.test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml similarity index 100% rename from qwen-endpoint-happy.dstack.yml rename to .test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml diff --git a/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml b/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml new file mode 100644 index 0000000000..e2b771c39a --- /dev/null +++ b/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml @@ -0,0 +1,9 @@ +type: endpoint +name: qwen-endpoint-log-smoke + +model: Qwen/Qwen3-0.6B +preset_policy: create + +backends: [runpod] +spot_policy: on-demand +max_price: 0.5 diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index 2dcc8c9a60..cf9675c976 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -133,13 +133,14 @@ Observed result on 2026-07-04: ### Preset Reuse Preview -After cleanup, `qwen-endpoint-happy.dstack.yml` was changed from `preset_policy: create` -to `preset_policy: reuse-or-create` so rerunning the smoke uses the saved preset first. +After cleanup, `.test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml` was changed +from `preset_policy: create` to `preset_policy: reuse-or-create` so rerunning the smoke +uses the saved preset first. Preview command: ```bash -echo n | uv run dstack apply -f qwen-endpoint-happy.dstack.yml +echo n | uv run dstack apply -f .test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml ``` Observed result on 2026-07-04: From 47c7d0f4d493a525a7020187a9c5c1f497626a3c Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 6 Jul 2026 13:53:14 +0200 Subject: [PATCH 13/21] Checkpoint endpoint preset reuse e2e --- .../qwen-endpoint-log-smoke.dstack.yml | 1 + .../no-preset-reapply.dstack.yml | 4 + endpoint-e2e-testing-report.md | 399 +++++++++ endpoint-implementation-plan.md | 825 ++++++++++++++++++ .../background-loop.md | 241 +++++ .../claude-agent-sdk.md | 407 +++++++++ .../config-models.md | 151 ++++ endpoint-implementation-research/critique.md | 63 ++ .../examples-llm-deployments.md | 129 +++ .../external-recipes.md | 85 ++ .../resource-lifecycle-blueprint.md | 201 +++++ .../run-submission-internals.md | 126 +++ .../service-runtime-health.md | 144 +++ .../settings-migrations-testing.md | 127 +++ .../skills-and-pr3856.md | 85 ++ .../verify-findings.md | 147 ++++ pyproject.toml | 6 + skills/dstack-prototyping/SKILL.md | 241 +++++ src/dstack/_internal/cli/commands/endpoint.py | 184 +++- src/dstack/_internal/cli/commands/logs.py | 62 +- src/dstack/_internal/cli/commands/preset.py | 66 -- src/dstack/_internal/cli/main.py | 2 - .../_internal/cli/services/completion.py | 7 - .../cli/services/configurators/endpoint.py | 62 +- src/dstack/_internal/cli/utils/endpoint.py | 26 +- src/dstack/_internal/cli/utils/preset.py | 61 +- .../_internal/core/models/endpoint_presets.py | 5 + src/dstack/_internal/core/models/endpoints.py | 10 +- .../background/pipeline_tasks/endpoints.py | 199 ++--- .../07_03_1243_03efb71a1563_add_endpoints.py | 11 +- src/dstack/_internal/server/models.py | 7 +- .../_internal/server/routers/endpoints.py | 45 +- .../server/schemas/endpoint_presets.py | 4 + .../_internal/server/schemas/endpoints.py | 2 +- .../server/services/endpoints/__init__.py | 70 +- .../services/endpoints/agent/__init__.py | 1 + .../server/services/endpoints/agent/claude.py | 179 +++- .../agent/resources/deployment_harness.md | 59 -- .../dstack_cli_and_service_authoring.md | 34 - .../agent/resources/recipes_guide.md | 25 - .../agent/resources/system_prompt.md | 42 +- .../server/services/endpoints/planning.py | 2 +- .../server/services/endpoints/presets.py | 111 ++- src/dstack/_internal/server/settings.py | 4 +- src/dstack/api/server/_endpoint_presets.py | 15 +- src/dstack/api/server/_endpoints.py | 8 +- .../_internal/cli/commands/test_endpoint.py | 108 +++ src/tests/_internal/cli/commands/test_logs.py | 99 +-- .../services/configurators/test_endpoint.py | 53 +- .../_internal/cli/utils/test_endpoint.py | 31 +- src/tests/_internal/cli/utils/test_preset.py | 24 +- .../pipeline_tasks/test_endpoints.py | 272 +++--- .../server/routers/test_endpoints.py | 148 +++- .../services/endpoints/test_claude_agent.py | 149 +++- .../server/services/test_endpoint_presets.py | 71 +- 55 files changed, 4712 insertions(+), 928 deletions(-) create mode 100644 .test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml create mode 100644 endpoint-e2e-testing-report.md create mode 100644 endpoint-implementation-plan.md create mode 100644 endpoint-implementation-research/background-loop.md create mode 100644 endpoint-implementation-research/claude-agent-sdk.md create mode 100644 endpoint-implementation-research/config-models.md create mode 100644 endpoint-implementation-research/critique.md create mode 100644 endpoint-implementation-research/examples-llm-deployments.md create mode 100644 endpoint-implementation-research/external-recipes.md create mode 100644 endpoint-implementation-research/resource-lifecycle-blueprint.md create mode 100644 endpoint-implementation-research/run-submission-internals.md create mode 100644 endpoint-implementation-research/service-runtime-health.md create mode 100644 endpoint-implementation-research/settings-migrations-testing.md create mode 100644 endpoint-implementation-research/skills-and-pr3856.md create mode 100644 endpoint-implementation-research/verify-findings.md create mode 100644 skills/dstack-prototyping/SKILL.md delete mode 100644 src/dstack/_internal/cli/commands/preset.py delete mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md delete mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md delete mode 100644 src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md create mode 100644 src/tests/_internal/cli/commands/test_endpoint.py diff --git a/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml b/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml index e2b771c39a..81e7afbb21 100644 --- a/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml +++ b/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml @@ -3,6 +3,7 @@ name: qwen-endpoint-log-smoke model: Qwen/Qwen3-0.6B preset_policy: create +max_agent_budget: 3.0 backends: [runpod] spot_policy: on-demand diff --git a/.test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml b/.test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml new file mode 100644 index 0000000000..7883ac5c66 --- /dev/null +++ b/.test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml @@ -0,0 +1,4 @@ +type: endpoint +name: endpoint-lifecycle-reapply-smoke +model: dstack/test-endpoint-no-preset +preset_policy: reuse diff --git a/endpoint-e2e-testing-report.md b/endpoint-e2e-testing-report.md new file mode 100644 index 0000000000..5914a94c31 --- /dev/null +++ b/endpoint-e2e-testing-report.md @@ -0,0 +1,399 @@ +# Endpoint E2E Testing Report + +Date: 2026-07-06 + +## Scope + +This report covers the first real endpoint agent e2e test against a local dstack server with a real RunPod-backed service. The goal was not to prove every endpoint feature, but to validate the most important path: + +1. Apply an `endpoint` configuration. +2. Let the server start the Claude Code based agent. +3. Let the agent use the real `dstack` CLI to create and debug candidate services. +4. Verify the final model endpoint with a real OpenAI-compatible request. +5. Return a final report to the server. +6. Link the endpoint to the final service run. +7. Save a reusable endpoint preset. +8. Mark the endpoint running. + +The test was run outside the repo in `~/dstack-endpoints-demo`. + +## Test Environment + +Project: `endpoint-e2e` + +Endpoint config: + +```yaml +type: endpoint +name: qwen-endpoint-e2e + +model: Qwen/Qwen3-0.6B + +backends: + - runpod +fleets: + - endpoint-e2e-runpod +spot_policy: on-demand +max_price: 0.5 + +preset_policy: create +max_agent_budget: 3 +``` + +Fleet config: + +```yaml +type: fleet +name: endpoint-e2e-runpod + +nodes: 0..1 + +resources: + gpu: 1 + disk: 100GB.. + +backends: + - runpod + +spot_policy: on-demand +max_price: 0.5 +idle_duration: 5m +``` + +Final service run: + +```text +run name: qwen-endpoint-e2e-2 +run id: ea8597ff-4567-418f-9d60-c620991d79a5 +backend: runpod +region: EU-RO-1 +gpu: NVIDIA RTX 2000 Ada Generation, 16GB +price: $0.24/hr +image: vllm/vllm-openai:v0.11.0 +model: Qwen/Qwen3-0.6B +``` + +Final endpoint state: + +```text +qwen-endpoint-e2e running qwen-endpoint-e2e-2 +``` + +Saved preset: + +```text +qwen-qwen3-0-6b-ea8597ff +``` + +Preset path: + +```text +~/.dstack/server/projects/endpoint-e2e/presets/qwen-qwen3-0-6b-ea8597ff.dstack.yml +``` + +## What Worked + +The endpoint apply path successfully created an endpoint record and moved it into the server-side background processing loop. + +The server started the Claude Code agent from the endpoint worker and passed endpoint/project constraints into the agent context. + +The agent used real `dstack` CLI commands, not server helper APIs, to submit candidate services and inspect their state. + +The agent recovered from a failed first serving attempt, selected a different image, submitted a second candidate service, waited for it to run, and verified the model endpoint with a real `/v1/chat/completions` request. + +The final verification was strong enough for v0: HTTP 200, OpenAI-compatible response shape, non-empty generated content, and response model matching `Qwen/Qwen3-0.6B`. + +The server consumed `final_report.json`, found the reported run, verified it was a service owned by the same user/project, linked it through `service_run_id`, saved a preset, and marked the endpoint `running`. + +Endpoint logs now show concise agent progress rather than raw service logs for this run: + +```text +research: Resuming: prior run qwen-endpoint-e2e-1 failed... +candidate: Submitting service qwen-endpoint-e2e-2... +verification: Verified: real chat completion... +done: Service qwen-endpoint-e2e-2 verified... +``` + +The saved preset contains both reusable service requirements and actual tested hardware. For this run, reusable requirements are broad enough for matching: + +```text +cpu=2.. mem=8GB.. disk=100GB gpu=16GB..:1 +``` + +The preset also stores tested hardware separately: + +```text +cpu=6 mem=31GB disk=100GB gpu=RTX2000Ada:16GB:1 +``` + +## What Failed First + +The first candidate run used a bad final service image: + +```text +vllm/vllm-openai:latest +``` + +That resolved to a newer vLLM/CUDA stack requiring a newer NVIDIA driver than the RunPod host exposed. The service failed with a driver/runtime mismatch. This was a useful failure: it proved the agent must not blindly use `latest` for final serving images. + +The initial agent behavior also used a candidate run name too close to the endpoint name. This caused CLI/log ambiguity and made it harder to distinguish endpoint logs from service logs. + +Earlier endpoint logs were too noisy in some cases: large CLI output, service logs, and trace-like content could leak into user-facing endpoint logs. The current run is better, but the logging contract still needs hardening. + +The endpoint stayed in `clauding` after the service was already healthy because the server intentionally waits for the agent's `final_report.json`. In this run: + +```text +verification.json 10:07:15 +final_report.json 10:07:51 +endpoint running after worker consumed final_report.json +``` + +This is correct behavior, but the UX needs clearer progress messages between "service is healthy" and "endpoint is running". + +## Fixes Made During Testing + +Agent prompt now requires candidate service run names to be distinct from the endpoint name. Suggested naming is short attempt-based names such as: + +```text +-1 +-2 +``` + +Agent prompt and `dstack-prototyping` skill now warn against unproven `:latest` images for final serving and emphasize image/runtime/driver compatibility. + +The backend value passed into the agent prompt was fixed to use normal lowercase values such as `runpod`. + +The prompt now tells the agent not to wait only on log text. It should poll structured run state and break on terminal statuses. + +The agent flow now asks for normal logs first, using debug logs only when needed. + +Endpoint status naming is now `running` for successful endpoints. The early `active` prototype rows are cleaned directly in the local DB; the runtime no longer carries an `active` alias. + +Endpoint preset storage was moved under the project-scoped server directory: + +```text +~/.dstack/server/projects//presets +``` + +The preset model was adjusted to separate reusable requirements from tested resources. + +Endpoint UX was split from run UX: + +- endpoint progress logs use `dstack endpoint logs `; +- endpoint lifecycle stop uses `dstack endpoint stop `; +- endpoint presets use `dstack endpoint preset ...`; +- top-level `dstack logs` and `dstack stop` remain run-only; +- the old top-level `dstack preset` command was removed. + +Focused agent tests pass: + +```text +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py +17 passed +``` + +## Preset Reuse Status + +A reuse preview was run with: + +```yaml +type: endpoint +name: qwen-endpoint-e2e-reuse + +model: Qwen/Qwen3-0.6B + +backends: + - runpod +fleets: + - endpoint-e2e-runpod +spot_policy: on-demand +max_price: 0.5 + +preset_policy: reuse +``` + +The planner did match the saved preset: + +```text +Preset qwen-qwen3-0-6b-ea8597ff +``` + +But it did not find an available offer while the original service was still using the `nodes: 0..1` fleet: + +```text +No matching instance offers available. +``` + +So preset matching is proven at plan level, but actual second deployment from the saved preset is not yet proven. To test it properly, we should either stop the current endpoint/service first or use a fleet that allows another node. + +## Remaining Issues + +The agent harness is still early. It completed one real model deployment, but we should not generalize too much from a small Qwen/vLLM case. + +Endpoint user logs need a stricter contract. They should contain major agent milestones only. Raw command traces, large YAML, huge offer tables, service logs, debug logs, and sensitive config output must stay out of normal endpoint logs. + +The first progress line is still too late. The user should see early progress immediately after the agent starts: what it is checking, what constraints it sees, and whether it is researching, submitting, waiting, or verifying. + +`clauding` can look stuck when the service is already healthy but the agent is still verifying or writing the final report. We need better progress events, not necessarily another status. + +The agent still needs much better deployment judgment. The `latest` image failure was caught and recovered from, but the next iterations should make image selection, driver compatibility, model memory sizing, and framework choice more systematic. + +The current e2e did not exercise dev-environment based prototyping. That is central to the long-term endpoint idea, especially for harder models, custom recipes, multi-node setups, and performance-sensitive serving. + +The current e2e did not test multi-replica services, replica groups, autoscaling, PD disaggregation, gateways, private models, or larger models. + +The current e2e did not test server restart while Claude is still running. We have code paths for resuming from workspace artifacts, but this needs a real restart test. + +The current e2e did not test multiple server instances with Postgres. The endpoint worker uses the existing lock/pipeline pattern, but the multi-server case still needs an explicit integration test. + +The current e2e did not test budget interruption behavior. `max_agent_budget` is passed/configured, but we still need a real test proving accumulated agent spend is tracked per endpoint provisioning attempt and interrupts safely. + +Debug traces may still contain sensitive command output if the agent runs unsafe inspection commands. This needs a concrete redaction policy before broader testing. + +Cleanup behavior after failed candidate services still needs pressure testing. The agent recovered from `qwen-endpoint-e2e-1`, but cleanup policy for abandoned candidate runs needs to be clear and observable. + +## 2026-07-06 Preset Reuse E2E + +Goal: reapply the stopped `qwen-endpoint-e2e` endpoint with `preset_policy: reuse` from +`~/dstack-endpoints-demo`, prove that the saved preset can create a new service without Claude, and +verify that endpoint lifecycle stays coherent. + +Pre-launch endpoint row: + +```text +id: fa60bac3-bd16-47bf-9c1a-768d5c5025aa +status: stopped +run: - +``` + +Pre-launch run state: + +```text +qwen-endpoint-e2e-2 is still visible in `dstack ps -v`, but it is stopped. +``` + +Observation: keeping the stopped service run in normal run history is consistent with run UX, but +endpoint reuse must not treat it as live ownership. The endpoint row is the source of truth for the +current backing service; old stopped runs are history. + +Launch result: + +```text +Configuration: qwen-endpoint-e2e-reuse-same-name.dstack.yml +Preset: qwen-qwen3-0-6b-ea8597ff +First offer: runpod EU-RO-1, NVIDIA RTX 2000 Ada Generation, $0.24/hr +``` + +Immediate post-launch state: + +```text +endpoint id: fa60bac3-bd16-47bf-9c1a-768d5c5025aa +status: provisioning +service run: qwen-endpoint-e2e-serving +service run id: 6a8f5e0a-cca6-4ae9-942b-aa5625d109ef +preset policy: reuse +``` + +Observation: terminal reapply reused the same endpoint row and reset `created_at` to the new +submission time. This matches the current "one row per endpoint name" direction. The saved preset +path produced a normal service run without invoking Claude. + +Progress at about 50 seconds: + +```text +endpoint: provisioning +run: qwen-endpoint-e2e-serving provisioning +logs: no endpoint log lines +``` + +Observation: empty endpoint logs are fine for preset reuse because no agent is involved. The only +UX gap is apply-time feedback while the CLI is detached: users see `provisioning`, but detailed +progress lives in normal run status/events/logs rather than endpoint logs. + +Progress at about 2-3 minutes: + +```text +instance: runpod EU-RO-1, instance id 4zy2c99vxg9gyl, status BUSY +job: still PROVISIONING +logs: service logs empty +``` + +Observation: offer matching and RunPod instance creation worked. The slow part is after the instance +is available, while the job/service is not yet registered. For real UX, endpoint progress should +distinguish "waiting for capacity", "starting instance", "pulling image", and "waiting for service +probe" when dstack already has those signals. + +Progress at about 3-4 minutes: + +```text +run: running +probe: failing +endpoint: provisioning +logs: first vLLM startup line appears +``` + +Observation: the preset path is not marking the endpoint running just because the service process is +running. It waits for the service/probe signal, which is the right minimum behavior for preset reuse. + +Final preset-reuse state: + +```text +13:42:59 run qwen-endpoint-e2e-serving RUNNING +13:45:24 service replica registered to receive requests +13:45:32 endpoint qwen-endpoint-e2e PROVISIONING -> RUNNING +url: /proxy/services/endpoint-e2e/qwen-endpoint-e2e-serving/v1 +``` + +Result: preset reuse works end to end. It reused the same endpoint row, created a service from the +saved preset without Claude, waited for the service probe/registration signal, and marked the +endpoint `running` with the service proxy URL. + +Observation: the runtime took about five minutes from endpoint apply to endpoint running. Most of +that was normal cold start: RunPod instance provisioning, vLLM image/container startup, model +download, torch compile, CUDA graph capture, and service registration. This is a useful baseline +for future agent/preset comparisons. + +Direct endpoint verification: + +```text +POST /proxy/services/endpoint-e2e/qwen-endpoint-e2e-serving/v1/chat/completions +status: 200 +model: Qwen/Qwen3-0.6B +``` + +Observation: the proxy endpoint is usable after endpoint status becomes `running`. The small Qwen +model did not follow an "exactly reply" instruction cleanly because it emitted reasoning text, so +future verification prompts for this model should check API/model correctness and non-empty content +rather than strict literal instruction following. + +Stop result: + +```text +13:46:42 endpoint marked for stopping +13:46:52 service run RUNNING -> TERMINATING +13:47:06 job TERMINATED and RunPod instance TERMINATED +13:47:18 endpoint STOPPED +``` + +Result: `dstack endpoint stop` stopped the backing service run, terminated the RunPod instance, and +left the endpoint visible as `stopped`. + +## Assessment + +This was the first meaningful proof that the endpoint idea can work end to end: endpoint config in, real agent work, real service deployed, real model request verified, endpoint running, preset saved. + +The biggest result is not that Qwen 0.6B runs. The important result is that the system survived a real failed candidate, recovered through agent reasoning, and produced a verified final service that the server could link and persist. + +The biggest remaining risk is still the harness quality. For v1, the server plumbing is useful only if the agent loop becomes strong at model/framework/hardware reasoning, uses dstack efficiently, keeps logs readable, and leaves enough trace evidence to improve each failed run. + +## Next Tests + +1. Run another small no-preset agent deployment and compare behavior against the first Qwen run. +2. Run a server restart test while the endpoint is still `clauding`. +3. Test endpoint stop while Claude is running and after a candidate run has been submitted. +4. Test budget interruption once spend is persisted per endpoint provisioning attempt. +5. Run a private/Hugging Face gated model test with environment handling. +6. Run a larger common model after confirming budget and hardware. +7. Test one dev-environment prototyping flow where the agent experiments before submitting the final service. +8. Add log/trace redaction tests. +9. Add regression tests for run-name ambiguity, final-report-to-running delay, and preset reuse. diff --git a/endpoint-implementation-plan.md b/endpoint-implementation-plan.md new file mode 100644 index 0000000000..fe4fa8f258 --- /dev/null +++ b/endpoint-implementation-plan.md @@ -0,0 +1,825 @@ +# Implementation plan: `endpoint` configuration type + +Status: **implementation in progress**. This document started as a research-backed implementation plan verified against `master` @ `28ea5f86f` (2026-07-03); it is now also used as the implementation tracker. Line numbers drift; symbol names are the anchor. + +## Project thesis + +The endpoint feature is not primarily a new CRUD resource. It is a **learning deployment loop**: + +1. A user asks dstack to serve a model. +2. If dstack already has a tested preset for that model, it should submit a normal dstack service from that preset and mark the endpoint running when the service is ready. +3. If dstack does not know a working deployment yet, a real agent investigates, uses dstack like a power user, deploys and verifies a final service, and saves what worked as a preset. +4. The next compatible endpoint for that model should be able to use the preset path without paying the agent again. + +The preset path and the agent path must converge. If the agent can make a model work once but saves a preset that is too narrow, unreproducible, missing evidence, or unusable without the agent, the feature has not solved the real problem. + +Non-negotiable project invariants: + +- A preset is a verified deployment artifact, not just a service YAML. +- `resources` are scheduling requirements used for reuse/offer matching; `tested_resources` are exact verified hardware evidence. +- The server owns endpoint state, locking, run linking, stopping/cleanup, and preset persistence. The agent owns deployment investigation and final functional verification. +- The server must not mark an agent-created endpoint running based only on its own service-readiness bookkeeping; the agent must report that the requested model was actually served and answered a real model request. +- Work should be driven by real deployment failures. Avoid adding abstractions before live runs show they are needed. + +## 0. Implementation checklist and plan deltas + +Keep this section current while implementing. It is the short operational view of the longer plan below: what is already in the working tree, what is still missing for v1, and where the implementation intentionally diverged from the original plan after code/UX review. + +### Done + +- [x] Research artifacts copied into the repo under `endpoint-implementation-research/`. +- [x] Endpoint core/API/CLI skeleton: `type: endpoint`, `EndpointConfiguration`, endpoint REST schemas/router/client group, `dstack endpoint list|get|logs|stop|preset`, and `dstack apply` support. +- [x] Endpoint DB/pipeline skeleton: `EndpointModel`, migration, pipeline registration, durable locking, fetch/worker flow, events, and status transitions. +- [x] Apply-plan UX aligned with run apply: one stable property table, preset policy, selected preset/offers when present, no `Model`/`Action`/`Service`/`Agent`/custom provisioning section. +- [x] Endpoint plan displays the configured/default `preset_policy` (`reuse-or-create` by default), not the effective fallback path. +- [x] Preset-backed provisioning path: local presets are matched through normal run planning, service runs are submitted through the existing run apply path, and `creation_policy` is not forced to `reuse`. +- [x] Preset planning distinguishes a model-matched preset from a provisionable preset. Preset submission requires available offers for every planned job; `reuse-or-create` can fall through to the agent path when only non-provisionable presets match. +- [x] Endpoint lifecycle safety: conflict-checked submission, server-side stop that stops the linked backing run, and failed-endpoint teardown of the linked non-terminal run. +- [x] Endpoint apply treats any terminal same-name endpoint as finished: no stop prompt; create resets the same endpoint row back to `submitted`, and only non-terminal same-name endpoints require stop/override. +- [x] Preset-backed endpoint submission handles its deterministic service-run-name conflicts like run apply: non-terminal conflicting runs fail before submission, while terminal conflicting runs are left to the existing run submission path to recycle. +- [x] Endpoint readiness uses backing service readiness only: run is `RUNNING`, has a registered running job, and exposes `ServiceSpec.model.base_url`; no extra endpoint probe in v1. +- [x] Preset storage/parsing: `EndpointPresetService`, local-dir implementation, `endpoint-preset` YAML wrapper, ordered `replica_spec_groups`, validation, atomic no-overwrite save, env value redaction, and literal non-secret env preservation. +- [x] Learned preset plumbing: build a preset from a ready service run, record replica spec groups in service order, preserve the number of currently registered running replicas per group, and save it when an agent-provisioned endpoint becomes running. +- [x] Endpoint preset CLI: `dstack endpoint preset list|get --json|delete`. The compact list shows scheduling resources used for reuse/offer matching; exact verified hardware stays in `tested_resources` and is exposed by `get --json`, not the compact list. +- [x] Endpoint status names adjusted: `clauding` while the server agent investigates/deploys, `running` for a ready endpoint, `stopping`/`stopped` for endpoint stop. The runtime `active` alias was removed; local prototype DB rows should be cleaned directly. +- [x] Endpoint UX split from run UX: `dstack endpoint logs` reads endpoint progress logs, `dstack endpoint stop` stops endpoints, top-level `dstack logs`/`dstack stop` remain run-only, and top-level `dstack preset` was removed. +- [x] Running endpoint apply output renders known relative proxy URLs as absolute URLs using the configured API server URL. +- [x] Endpoint migration is safe for Postgres partial indexes and boolean server defaults. +- [x] Agent lifecycle skeleton: `AgentService` abstraction, disabled default, agent settings, `EndpointPlan.provisioning_plan=type:"agent"` when an enabled agent is injected, fake-agent pipeline integration, and a v0 Claude Code subprocess runtime. +- [x] Agent unavailable UX distinguishes a genuinely missing `DSTACK_AGENT_ANTHROPIC_API_KEY` from a configured key with no real agent implementation registered yet. +- [x] Endpoint run identity uses `EndpointModel.service_run_id` for the current/latest service run and `EndpointRunSubmissionModel` for ordered endpoint-submitted run history. +- [x] Structured final-report validation for the endpoint pipeline contract. +- [x] Endpoint-scoped workspace/process handling added as part of the v0 Claude Code runtime, not as standalone scaffolding. +- [x] Raw Anthropic Messages-loop prototype was removed. It was the wrong abstraction; the endpoint agent must use a real agent runtime, not a hand-rolled tool loop. +- [x] Packaged endpoint-agent prompt/context resources: `resources/system_prompt.md` plus repo-root `skills/dstack` and `skills/dstack-prototyping`, force-included into wheels/sdists and copied into each Claude workspace. +- [x] First full learning-loop proof: Claude-created Qwen preset was reused later by `preset_policy: reuse` without invoking Claude; the endpoint reached `running`, answered a model request, and stopped cleanly. +- [x] Focused endpoint tests and static checks currently pass (`pytest` endpoint suites, `ruff`, `pyright`). + +### Still open for v1 + +- [ ] Harden the real Claude Code subprocess runtime based on live failures: restart/resume semantics, stop-time cancellation, duplicate-process prevention, packaging, and live-run efficiency. +- [ ] Finish runtime durability: persisted usage accounting, restart-safe handoff from agent report to linked service run, endpoint stop abort handling, and cleanup of non-final candidate runs. +- [ ] Guard learned-preset quality: scheduling `resources` must stay broad enough for reuse, exact hardware must remain in `tested_resources`, and saved presets must be usable without the agent. +- [ ] Wire structured agent handoff into preset provenance: final candidate run name/id, final service YAML/content, recipe sources, verification summary, and failure summary. +- [ ] Runtime recipe grounding against vLLM recipes / SGLang docs / HF model cards, with mocked zero-network tests. +- [ ] Foreground endpoint apply following server-side endpoint logs/status without attach. +- [ ] Endpoint preset inspect polish: keep `dstack endpoint preset get --json` useful enough for exact `tested_resources`, provenance comments/metadata, and service recipe review without cluttering the compact list output. +- [ ] Real agent runtime dependencies installed automatically by normal server install/deploy paths: local `uv` server installs and server Docker images must include the runtime; no manual post-install dependency step. +- [ ] Endpoint update/version design before in-place updates: a `configuration_version`/deployment guard so stale background workers in multi-server Postgres setups cannot mark a newer endpoint config running. +- [ ] Documentation: endpoint reference schema page, env vars, concepts page, manual e2e runbook, example presets. +- [ ] Real no-preset agent e2e with a small model and budget-confirmed hardware. + +### Critical assessment + +The endpoint storage/status/preset/run-link side is becoming reasonable. The feature is still not trustworthy because the agent loop and learned-preset quality are not yet proven across repeated real deployments. + +Highest current risks: + +- Agent works once but saves a preset that is too exact, stale, missing provenance, or not reusable. +- Agent wastes budget because hardware selection, offer inspection, or failure recovery is weak. +- Agent logs/traces are either too noisy for users or too thin for debugging. +- Server restart/stop during `clauding` can leave ambiguous state or orphan candidate runs. +- Duplicate learned presets accumulate without a repair/update policy. +- We overfit code around the first Qwen smoke runs before testing enough real model/backends/failure modes. + +### Immediate next steps: repeat the loop, then harden + +Do these in order. Do not add more harness surface area until the previous step has produced evidence. + +1. **Audit the reused preset after the run.** Confirm the preset's `resources` are scheduling requirements, `tested_resources` are exact evidence, the service recipe is sufficient, and offer matching is not over-narrow. If reuse fails later, fix the preset contract/builder before touching the agent prompt. +2. **Run one fresh no-preset agent deployment with a small model and a confirmed budget.** Capture command count, elapsed time, spend, hardware choice, service YAML, dstack plan output, candidate cleanup, progress logs, final report, and saved preset. +3. **Harden only from observed failures.** Fix concrete problems in prompt/context, endpoint logs, final report validation, preset saving, run linking, or cleanup. Avoid speculative tools/functions. +4. **Add durable agent budget accounting before retries/resumes.** `max_agent_budget` must cap total spend per endpoint provisioning attempt, not just a single subprocess invocation. +5. **Harden restart/stop while clauding.** Stop should abort the live process and stop linked/candidate runs using explicit submission records. Restart should reconcile existing workspace, final report, and submissions before launching anything new. +6. **Move to a more common/interesting model only after the small loop repeats.** Confirm hardware and budget before trying larger models. Use those runs to evolve hardware-selection behavior and recipe grounding. + +### How to overcome the current risks + +Treat this as a working hypothesis, not a fixed design. After each real endpoint run, update the plan with what actually happened and remove or change tactics that did not help. + +| issue | realistic mitigation | evidence to collect | reconsider if | +|---|---|---|---| +| Learned preset works once but is not reusable | Run an immediate preset-reuse test after every successful agent deployment. Validate that the saved `service` + `resources` can be planned/submitted without the agent and that `tested_resources` are only evidence. | apply plan path selected, offers shown, final service run, endpoint status, model request result, saved preset YAML before/after | the preset path fails for reasons unrelated to transient capacity; then fix preset contract/builder before prompt changes | +| Preset `resources` are too exact or too loose | Keep `resources` sourced from the final service config, but add validation/reporting that flags exact GPU names or full exact CPU/memory/disk constraints in learned presets unless explicitly justified by the service shape. Do not auto-generalize blindly; first inspect the agent's service YAML and the run plan. | final service YAML, saved `resources`, saved `tested_resources`, offer count before/after, reason for any exact GPU/model constraint | exact constraints repeatedly block reuse, or broad constraints repeatedly schedule hardware that cannot serve the model | +| Agent chooses poor hardware or wastes attempts | Give the agent a required decision trace: model sizing evidence, candidate hardware envelope, current dstack offers, planned service resources, and why it is submitting now. Start with prompt/context; only add server-side enforcement after observed bad behavior is clear. | recipe/model sources, offer table excerpt, chosen resources, rejected alternatives, command count, elapsed time, GPU spend | the trace is absent or useless in two real runs; then enforce a structured pre-submit decision artifact before `dstack apply -y -d` | +| Agent logs are noisy or unhelpful | Split logs into two layers: endpoint log gets concise major events authored by the agent; workspace trace stores command output, YAML, raw logs, and final report. Do not put full CLI output or service logs in endpoint logs. | `dstack endpoint logs ` readability, workspace artifacts, ability to diagnose failure without server stdout | endpoint logs still duplicate/replay, hide important state, or force reading server logs for normal debugging | +| Restart during `clauding` duplicates work or loses the final run | Reconcile before launch: check existing process metadata, final report, endpoint run submissions, linked `service_run_id`, and live candidate runs. Do not start a new agent until reconciliation decides the previous attempt is unrecoverable. | restart test with a live process, restart test after final report before link, endpoint submissions, linked run, no duplicate paid process | reconciliation logic becomes fragile or still misses cases; then consider moving agent execution to a separately tracked durable task model | +| Stop during `clauding` leaks resources | Add process cancellation and candidate-run cleanup using `EndpointRunSubmissionModel`, not name heuristics. Endpoint stop should request abort, stop linked/submitted non-terminal runs, then finish through the normal run lifecycle. | stop during Claude process, stop after candidate run submitted, final run statuses, endpoint stopped, no running GPU run | cancellation depends on runtime behavior we cannot rely on; then isolate agent execution in a managed process/task with explicit supervisor state | +| Budget cap can be bypassed by restart/resume | Persist provider usage/spend per endpoint provisioning attempt and check remaining budget before starting another agent process. Until this exists, do not add automatic retries/resumes that can spend again. | Claude reported usage/cost, persisted attempt spend, remaining budget, refusal path when exhausted | Claude Code cannot expose reliable usage data; then use a stricter wall-clock/process cap plus operator-visible warning until runtime changes | +| Duplicate learned presets accumulate | Keep append-only for v1, but add inspection/deletion and a simple duplicate report. Do not auto-update until the harness can prove an older preset is not reproducible under its claimed conditions. | duplicate model/resource list, preset source endpoint/run, reuse success/failure per preset | duplicates make selection unstable; then add deterministic ranking or explicit user-selected preset before adding automatic update | +| Agent harness design overfits early Qwen runs | Maintain a scenario ladder: one tiny public model, one slightly larger common model, one gated/HF-token model, one no-offers/constraint case, one failure-cleanup case. Change harness only after comparing patterns across at least two scenarios unless the bug is clearly blocking. | per-scenario run notes, failure categories, command counts, spend, preset reuse result | the first two scenarios expose contradictory needs; then pause and rework the harness contract rather than patching locally | + +Near-term implementation strategy: + +1. Prefer prompt/context and artifact requirements for the first two real runs. They are cheaper to change and reveal what the agent naturally does. +2. Add server-side validation only where the failure would be expensive or dangerous: wrong project/user run, missing service `model`, missing final verification, budget exhaustion, leaked non-terminal runs, invalid preset shape. +3. Keep every hardening change tied to a reproducible trace: command transcript, endpoint log, final report, saved preset, and dstack run state. +4. After each real run, classify the issue as one of: dstack/backend provisioning bug, agent decision bug, prompt/context gap, preset contract bug, lifecycle/recovery bug, or UX/logging bug. Fix the right layer; do not let agent failures hide dstack/backend failures. +5. Be willing to change runtime approach if Claude Code cannot reliably run non-interactively, expose usage, cancel, preserve artifacts, or package cleanly in server installs/Docker. + +### Adjusted from the original plan + +- Preset service responsibility was narrowed to **storage only**. Matching, planning, and service apply live outside `EndpointPresetService`. +- Presets now store `replica_spec_groups` as ordered groups of `ResourcesSpec`, separate from the service config. Group order must match `ServiceConfiguration.replica_groups`; the implicit no-replica-groups case uses group `"0"`. +- Presets do **not** store backend, region, or instance type in v1. They store the tested resource topology that can be replanned against current fleets. +- A learned preset records how many registered running replicas existed per replica group at save time. Autoscaling metrics, scaling validation, and benchmark metadata are Later. +- Preset matching does **not** force `creation_policy: reuse`; default `reuse-or-create` may use elastic fleets that can provision new instances. +- Endpoint plan `preset_policy` remains the configured/default policy; the selected path (`preset`, `agent`, or `none`) is represented only by `provisioning_plan`. +- Endpoint config changes are handled like current run UX: prompt to stop/override, with the backing service cleanup performed server-side. In-place update/rolling redeploy is Later. +- Later endpoint config updates must reuse the existing service-run `get_plan`/`apply_plan`/rolling deployment machinery, and endpoint DB updates must be guarded by a configuration/deployment version for multi-server safety. +- Stop/override applies only to non-terminal existing endpoints. Terminal endpoints are replaceable like finished runs. +- Serving-run-name lookup is a conflict check for non-terminal runs only. Terminal conflicting runs follow normal run submission semantics: the run apply path can delete/recreate them, while endpoint code must not adopt/delete unrelated runs by name. +- No generic server-side endpoint health probe in v1. Existing service probes/registration are the server readiness source. In the agent path, the agent itself must perform final functional verification before reporting a final candidate. +- Automatic provisioning retry is Later. A failed endpoint is terminal for v1. +- Frontend UI is explicitly Later; no UI is required for v1. +- `DSTACK_AGENT_ANTHROPIC_API_KEY` remains the official server-agent env var. A local `DSTACK_ANTHROPIC_API_KEY` must be mapped/exported to that name if used. +- The raw `anthropic` Messages API loop is explicitly rejected for the real agent path. The next implementation must use a real agent runtime and package its dependencies automatically for server installs and Docker images. + +--- + +## 1. What we're building + +A new top-level dstack configuration type `endpoint`: the user declares *what model* they want served; dstack figures out *how* — either from a locally stored preset (one tested service deployment topology on ordered tested replica spec groups) that matches the project's existing fleets, or by launching an LLM agent (Anthropic) that authors and deploys a dstack service for that model. The endpoint follows the backing service lifecycle instead of adding a parallel serving layer. + +```yaml +type: endpoint +name: qwen3-32b # optional; auto-generated if omitted +model: Qwen/Qwen3-32B # required; HF model id / model name +env: + - HF_TOKEN # merged into whatever service gets deployed +# plus any ProfileParams: backends, regions, spot_policy, max_price, fleets, idle_duration, ... +``` + +``` +dstack apply -f endpoint.dstack.yml + └─ endpoint: SUBMITTED + └─ server background pipeline picks it up (multi-replica safe) + ├─ preset matches existing fleets? → submit service run from preset + ├─ else, agent enabled (DSTACK_AGENT_ANTHROPIC_API_KEY)? → agent deploys a service + └─ else → FAILED ("no matching preset found and server agent disabled") + └─ endpoint: CLAUDING (agent investigates/deploys, for no-preset agent path) + └─ endpoint: PROVISIONING (service run starting, model pulling, service probes passing) + └─ endpoint: RUNNING (service has a registered running job and model URL) +``` + +The endpoint's deliverable is an OpenAI-compatible base URL (from the backing service's `ServiceSpec.model.base_url`). + +### v1 scope (this plan) + +- New configuration type + DB model + REST API + CLI (`dstack apply`, `dstack endpoint list|get|logs|stop|preset`). +- `EndpointPipeline` background processing: SUBMITTED → CLAUDING/PROVISIONING → RUNNING/FAILED, STOPPING → STOPPED, crash recovery, and ownership-safe reconciliation. +- Preset subsystem: `EndpointPresetService` interface + local-directory implementation for storing/loading presets; endpoint planning code separately matches loaded presets against existing fleets, including elastic fleets that can provision new instances, via the run planner; successful agent deployments are saved back as sanitized local presets. +- Agent subsystem: `AgentService` interface + real Claude agent runtime implementation, a CLI-first execution workspace that lets the agent use the real `dstack` binary, vendored prompt/context, recipe/hardware grounding, and a minimal structured handoff for the final candidate. Raw LLM API loops are not accepted for v1. +- Endpoint readiness follows the backing dstack service lifecycle on the server side: a service run is usable once it is RUNNING, has a registered running job, and exposes a model URL. The agent path additionally requires the agent to verify the final candidate with a model request before reporting success; the server does not add a generic duplicate endpoint probe in v1. +- `DSTACK_AGENT_ANTHROPIC_API_KEY` + related settings; real agent runtime dependencies installed automatically by normal server install and Docker deployment paths. +- Endpoint apply/log UX: `dstack apply -f endpoint.dstack.yml` shows an advisory plan and confirmation first, then follows endpoint progress by polling server-side status/log storage; `dstack endpoint logs ` shows endpoint agent/progress logs, while backing service logs remain available via top-level `dstack logs `. No frontend UI in v1. + +Everything else → §12 "Later". + +--- + +## 2. Ground truth: existing machinery we build on + +These are the load-bearing facts a developer needs; each was verified in code. + +1. **Configuration registration hub** — `src/dstack/_internal/core/models/configurations.py`: `ApplyConfigurationType` enum, `AnyApplyConfiguration`, `BaseApplyConfiguration.__root__` (discriminated on `type`), `AnyDstackConfiguration` (feeds the editor JSON schema built in CI). CLI dispatch via `apply_configurators_mapping` in `src/dstack/_internal/cli/services/configurators/__init__.py`. There is **no generic server-side apply endpoint** — every resource type has its own typed router + `APIClient` group. +2. **Non-run resource blueprint** — gateways and volumes show the durable pipeline pattern: model with `PipelineModelMixin` (`lock_expires_at`/`lock_token`/`lock_owner`, `models.py:204`) + `status`/`status_message`/`last_processed_at`; status enum in core models; service module (`services/volumes.py` is the cleanest); `Pipeline` subclass registered in `PipelineManager.__init__` (`background/pipeline_tasks/__init__.py:35-48`); router + schemas; configurator. Gateways/volumes also use an async delete flag, but endpoints intentionally do not expose delete in v1; endpoint lifecycle is stop/history. +3. **Multi-replica safety** is NOT the in-memory locker (`services/locking.py` — no-op `DummyResourceLocker` on Postgres by design). It is: durable lock columns claimed in the fetch transaction with `SELECT ... FOR UPDATE SKIP LOCKED`, a `Heartbeater` that checks tracked items every ~1s and extends `lock_expires_at` whenever a lease comes within `heartbeat_trigger` of expiry (so with 30s timeout / 15s trigger, ~every 15s), and every subsequent write guarded by `WHERE id = :id AND lock_token = :token`. If a replica dies, the lease expires (≤ ~30s) and another replica's fetcher re-claims the row (`or_(lock_expires_at.is_(None), lock_expires_at < now)`, ordered by `last_processed_at ASC`). Long operations are legitimate: the heartbeater extends the lease indefinitely (`instances/cloud_provisioning.py` runs minutes-long provisioning inside one `process()` call), but crash recovery restarts the whole step — so steps must be reconcile-first/idempotent. +4. **Server-side run creation is first-class** — `src/dstack/_internal/server/services/runs/__init__.py`: `submit_run(session, user, project, run_spec, pipeline_hinter=None) -> Run`, `stop_runs(...)`, `delete_runs(...)`, `get_run_by_name(...)`, `get_plan(session, project, user, run_spec, max_offers)`. Repo-less runs work: leave `run_spec.repo_id`/`repo_data` as `None` → `validate_run_spec_and_set_defaults` (`runs/spec.py`) fills the virtual repo (`DEFAULT_VIRTUAL_REPO_ID = "none"`) and `_get_run_repo_or_error` upserts the `RepoModel` row. Requires `run_spec.ssh_key_pub` or `user.ssh_public_key` set — and note `get_plan` calls the same validation, so this bites at *planning* time too. `submit_run` **commits the session multiple times** — always call it (and `stop_runs`/`delete_runs`) with a fresh `get_session_ctx()`, never inside a pipeline worker's token-guarded transaction. +5. **Run teardown is asynchronous and two-phase.** `stop_runs` only sets `termination_reason` + status `TERMINATING`; the RunPipeline performs actual termination. `TERMINATING` is **not** in `RunStatus.finished_statuses()` (`[TERMINATED, FAILED, DONE]`), and `delete_runs` raises `ServerClientError("Cannot delete active runs: ...")` for any non-finished run. Neither function raises for missing names — they filter and silently no-op. **Any "stop then delete" flow must wait (across pipeline iterations) for the run to finish before deleting.** +6. **Service readiness ≠ `RunStatus.RUNNING`.** A run is RUNNING when *any* replica job is RUNNING (`pipeline_tasks/runs/active.py`). The service-level readiness signal is `JobModel.registered`, set `True` only after all service probes pass (`_maybe_register_replica`, `jobs_running.py:1158`; it is set back to `False` only in the terminating pipeline, `jobs_terminating.py:787`). If `model:` is set and `probes` omitted, a default `POST {prefix}/chat/completions` probe is auto-generated (`services/jobs/configurators/base.py::_openai_model_probe_spec`). Endpoint v1 relies on this existing service mechanism instead of adding a parallel endpoint probe loop. +7. **In-process model probing exists but is not generic endpoint v1 behavior** — `_execute_probe` (`background/scheduled_tasks/probes.py:106-126`) POSTs `chat/completions` to a replica through `get_service_replica_client(job_model)` (`services/jobs/job_replica_http_client.py`, SSH tunnel + httpx over a Unix socket; URL host is a dummy `http://dstack`; the whole call is wrapped in `except (SSHError, httpx.RequestError) → False`). This is useful background for later agent verification/hardening, but v1 endpoint readiness should not duplicate service probes. +8. **Preset↔fleet matching primitive** — `_get_job_plan` (`services/runs/plan.py:970-988`): plan offers always include existing-instance offers, and add new-cloud offers only when `profile.creation_policy == CreationPolicy.REUSE_OR_CREATE` **and** `profile.instances is None`. This is exactly the "existing fleets" semantics we need: a non-empty available offer can mean either an already-idle instance or an elastic fleet that can create a new matching instance. If the user explicitly sets `creation_policy: reuse`, matching is restricted to currently existing instances; otherwise the default `reuse-or-create` allows elastic fleet capacity. (Caveats: each candidate evaluation can take seconds because the planner enumerates offers; and the per-job criterion under-checks total capacity for multi-replica configs — see §7.3.) +9. **No system user exists.** Only `admin` is guaranteed (`services/users.py::get_or_create_admin_user`, created at startup). Runs are only ever **soft-deleted** (`delete_runs` sets `RunModel.deleted=True`; nothing hard-deletes run rows except the project-cascade) — an `ondelete=SET NULL` on a run FK effectively never fires; handlers must treat `run.deleted == True` as "run gone". `RunModel` has no tags column; for endpoints, the serving run uses an explicit FK (`service_run_id`) and endpoint-submitted run history lives in `EndpointRunSubmissionModel`. Name/user/timestamp checks are not a durable ownership model. +10. **Packaging baseline**: hatchling normally packages `src/dstack` plus declared artifacts; repo-root `skills/` must be force-included explicitly or an installed server will not have them. This branch force-includes `skills/**` for wheels/sdists and keeps the endpoint-specific system prompt under `src/dstack`. **dstack pins `pydantic>=1.10.10,<2.0.0`** — do not import an agent runtime that requires pydantic v2 into the server process (§8.2). +11. **Agent skills live under repo-root `skills/`** — `skills/dstack/SKILL.md` and `skills/dstack-prototyping/SKILL.md` are the canonical skill files. `pyproject.toml` packages `skills/**` into wheels/sdists, and the endpoint harness locates that packaged `skills` directory before copying the required skills into each agent workspace's `.claude/skills`. +12. **A "templates" feature already exists** (`core/models/templates.py::UITemplate`, `services/templates.py`, `DSTACK_SERVER_TEMPLATES_REPO`) — UI-only, git-repo-sourced run-config templates. Presets must not collide with this namespace (we use "preset" consistently); reusing its git-repo fetch machinery is a Later option. +13. **External recipes**: vLLM recipes expose a machine-readable JSON API built for agents — `https://recipes.vllm.ai/models.json` (index, ~500 entries, dedupe on `derived_from`) + `https://recipes.vllm.ai//.json` (exact `vllm serve` command, docker image, per-GPU configs). SGLang Cookbook lives at `docs.sglang.io/cookbook` (source: `sgl-project/sglang:docs_new/cookbook`, MDX; discoverable via `https://docs.sglang.io/llms.txt`; pages fetchable raw by appending `.md`, but some embed commands in JSX). Both Apache-2.0 — runtime fetching and vendoring with attribution are fine. The old `sgl-cookbook` repo is archived read-only. +14. **Research notes behind this plan are now stored in-repo** under `endpoint-implementation-research/`, copied from `/private/tmp/claude-501/-Users-dstack-dstack/b621b4a9-b6ee-4b6b-813e-1084018def84/scratchpad/research/` plus `verify-findings.md`. These are planning artifacts, not packaged runtime resources; the packaged endpoint prompt and skills live under `src/dstack/_internal/server/services/endpoints/agent/` (§8.4). + +--- + +## 3. Configuration & core models + +New file `src/dstack/_internal/core/models/endpoints.py`: + +```python +class EndpointStatus(str, Enum): + SUBMITTED = "submitted" + PROVISIONING = "provisioning" + CLAUDING = "clauding" # server agent is investigating/deploying + RUNNING = "running" # backing service is ready + STOPPING = "stopping" + STOPPED = "stopped" + FAILED = "failed" + +class EndpointConfiguration(ProfileParams, generate_dual_core_model(EndpointConfigurationConfig)): + type: Literal["endpoint"] = "endpoint" + name: Optional[str] = None # dstack resource name; auto-generated if omitted + model: str # required; HF model id / served model name + env: Env = Env() + +class Endpoint(CoreModel): # API/runtime representation + id: uuid.UUID + name: str + project_name: str + user: str + configuration: EndpointConfiguration + created_at: datetime + status: EndpointStatus + status_message: Optional[str] + run_name: Optional[str] # backing service run (if provisioned) + url: Optional[str] # OpenAI-compatible base URL from ServiceSpec.model.base_url (read-time derived) + error: Optional[str] +``` + +Notes: +- `ProfileParams` (`core/models/profiles.py:310-493`) gives us `backends/regions/spot_policy/max_price/creation_policy/idle_duration/fleets/tags/...` for free — this is exactly the "profile params" requirement. Follow the run-config MRO pattern (`ProfileParams` first) and the pydantic-duality rule: never define `class Config` directly; create `EndpointConfigurationConfig(ProfileParamsConfig)` chaining `schema_extra` and pass it via `generate_dual_core_model(...)` as the last base (see `ServiceConfigurationConfig`, `configurations.py:1316`). +- Codebase is **pydantic v1** (`validator`/`root_validator`, `.json()`/`.parse_raw()`); do not write v2 APIs. +- No `EndpointSpec` type in v1 — `CreateEndpointRequest` takes a bare `configuration` (volumes parity, `CreateVolumeRequest`); introduce a spec wrapper together with the plugins work (§12) if/when needed. +- `Env` is a plain BaseModel with a custom root (`core/models/envs.py`) — bare `VAR` entries are `EnvSentinel`s resolved from `os.environ` **CLI-side only** (`ApplyEnvVarsConfiguratorMixin.apply_env_vars`). The server must reject configurations arriving with unresolved sentinels (validate in `create_endpoint`; `Env.as_dict()` raises `ValueError` on unresolved). +- Validate `name` with `validate_dstack_resource_name` (regex `^[a-z][a-z0-9-]{1,40}$`, `core/services/__init__.py`). `model` is deliberately a plain `str` in v1 (not `AnyModel`) — it's a requirement, not a service config field. + +**Registration checklist** (all in `core/models/configurations.py` unless noted): +1. Add `ENDPOINT = "endpoint"` to `ApplyConfigurationType` (~:1384). +2. Add `EndpointConfiguration` to `AnyApplyConfiguration` (~:1393). +3. Add it to `BaseApplyConfiguration.__root__` union (~:1401) — it's a "final configuration" (single `type` discriminator; no second-stage parse like volumes). +4. Add it to `AnyDstackConfiguration` (~:1440) — the editor JSON schema in CI (`.github/workflows/build-artifacts.yml`, `DstackConfiguration.schema_json()`) picks it up automatically. +5. Watch for circular imports: `configurations.py` already imports fleets/gateways/volumes models the same way. + +--- + +## 4. DB model, migration, events + +`src/dstack/_internal/server/models.py` — follow the normal pipeline model shape, but do not copy volume soft-delete semantics: endpoints have stop/stopped, not delete. + +```python +class EndpointModel(PipelineModelMixin, BaseModel): + __tablename__ = "endpoints" + + id: UUIDType(binary=False) pk, default=uuid.uuid4 + name: Mapped[str] = mapped_column(String(100)) + project_id: FK("projects.id", ondelete="CASCADE") + relationship + user_id: FK("users.id", ondelete="CASCADE") + relationship # endpoint creator; runs are submitted as this user + service_run_id: Mapped[Optional[uuid.UUID]] = FK("runs.id") + relationship + """The service run currently backing this endpoint (authoritative link; run name is convention only). + Runs are soft-deleted, so handlers must treat run.deleted == True as 'run gone' — no ondelete magic applies.""" + configuration: Mapped[str] = mapped_column(Text) # EndpointConfiguration JSON + status: Mapped[EndpointStatus] = mapped_column(EnumAsString(EndpointStatus, 100), index=True) + """Must be changed only via switch_endpoint_status() (API side) / token-guarded pipeline updates.""" + status_message: Mapped[Optional[str]] = mapped_column(Text) + provisioning_method: Mapped[Optional[str]] = mapped_column(String(100)) # "preset:" | "agent"; NULL until dispatched + created_at: NaiveDateTime, default=get_current_datetime + last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime) # MUST equal created_at on insert (fetch fast-path) + + __table_args__ = ( + UniqueConstraint("project_id", "name", name="uq_endpoints_project_id_name"), + Index("ix_endpoints_pipeline_fetch_q", last_processed_at.asc()), + ) +``` + +- One row per `(project_id, name)`. There is no endpoint soft delete in v1. Terminal same-name reapply resets the existing row and `EndpointRunSubmissionModel` preserves backing run submission history. +- JSON goes in `Text` columns as pydantic JSON — there is no JSON column type in this codebase. +- **Migration**: one new table, one migration. Generate with `cd src/dstack/_internal/server && alembic revision -m "add endpoints" --autogenerate`; lands under `migrations/versions/2026/` (per `alembic.ini` `file_template`). Plain `op.create_table` (a new table is additive ⇒ zero-downtime-safe; `contributing/MIGRATIONS.md`'s "separate migrations per table" rule concerns ALTERs of existing tables, and its `CREATE INDEX CONCURRENTLY` guidance applies to existing tables only). Column types: `sqlalchemy_utils.types.uuid.UUIDType(binary=False)`, `dstack._internal.server.models.NaiveDateTime()`. Remember tests create schema via `metadata.create_all`, so a *missing* migration won't fail unit tests — the migration is its own review item. +- **Events**: add `ENDPOINT` to `EventTargetType` (`core/models/events.py:12`) and an `EndpointModel` branch to `events.Target.from_model` (`server/services/events.py:89`) — `from_model` raises `ValueError` for unregistered types, so forgetting this crashes the first `emit()`. + +--- + +## 5. Server service module, REST API, CLI + +### 5.1 Service module — `src/dstack/_internal/server/services/endpoints/` (package: `__init__.py`, `presets.py`, `planning.py`, `agent/`) + +- `create_endpoint(session, project, user, configuration, pipeline_hinter) -> Endpoint`: + validate name + reject unresolved `EnvSentinel`s → name-uniqueness critical section (`lock_namespace = f"endpoint_names_{project.name}"`; SQLite: commit first + in-process lockset; Postgres: `pg_advisory_xact_lock(string_to_lock_id(ns))`) → duplicate non-terminal name ⇒ `ResourceExistsError` → duplicate terminal name ⇒ reset the same row to `SUBMITTED` with the new config, clear `service_run_id`/status message/provisioning method/lock fields, and refresh `created_at == last_processed_at == now` → auto-name via `generate_name()` loop if unset → insert row with `status=SUBMITTED`, `created_at == last_processed_at == now` → emit create/submit event → commit → `pipeline_hinter.hint_fetch(EndpointModel.__name__)`. +- `stop_endpoints(session, project, names, user)`: set `status=STOPPING` for non-finished endpoints, emit a stop event, commit, and `hint_fetch`. The pipeline owns stopping the linked backing service run and marking the endpoint `STOPPED`. +- `list_endpoints` (keyset pagination on `(created_at, id)` like volumes), `get_endpoint_by_name`, `endpoint_model_to_endpoint` (parse config via `EndpointConfiguration.__response__.parse_raw`; populate `run_name`/`url` at read time from the joined run's `service_spec` when present), `switch_endpoint_status(session, endpoint_model, new_status, actor=SystemActor())` + `emit_endpoint_status_change_event`. +- v1 does **not** call `apply_plugin_policies` — `ApplySpec` TypeVar (`dstack/plugins/_models.py:8`) doesn't include endpoints; extending it is a Later item (do not call the function with an unsupported type). + +### 5.2 REST API + +- `src/dstack/_internal/server/schemas/endpoints.py`: `ListEndpointsRequest`, `GetEndpointRequest{name}`, `GetEndpointPlanRequest{configuration, configuration_path?}`, `CreateEndpointRequest{configuration: EndpointConfiguration}`, `StopEndpointsRequest{names}` (all `CoreModel`). +- Endpoint configuration includes `preset_policy` (`reuse`, `create`, `reuse-or-create`, default `reuse-or-create`). This is distinct from profile `creation_policy`: `preset_policy` chooses whether endpoint provisioning may reuse a tested service recipe or ask the server agent to create one; `creation_policy` still controls whether the resulting service can reuse existing instances or provision new ones from elastic fleets. +- New core model `EndpointPlan`: `project_name`, `user`, `configuration`, `configuration_path`, `current_resource`, `action`, `preset_policy`, and `provisioning_plan`. `preset_policy` is the configured/default policy shown in the CLI; the selected path (`preset`, `agent`, or `none`) belongs in `provisioning_plan`. +- `provisioning_plan` is a small discriminated union: + - `type="preset"`: a **provisionable preset** was selected. This includes `preset_name`, `service_name`, `replica_spec_groups: list[EndpointPlanReplicaSpecGroup]`, and `job_offers: list[EndpointPlanJobOffers]` derived from `runs.get_plan` (enough for the CLI to print the selected preset, stable run-like scheduling properties, and first matching offers, without dumping the full run plan). + - `type="agent"`: no provisionable preset was found and the policy allows creation. The plan may include a short reason such as "matching preset qwen has no available offers"; the initial plan must not invent final offers because the agent may explore hardware and service shapes within endpoint/profile constraints. + - `type="none"`: no provisionable path is available, with the exact failure message that the pipeline will use (for example, `preset_policy: reuse` with no matching preset or only no-offer presets, or `preset_policy: reuse-or-create` with no provisionable preset and no usable server agent). +- Preset terminology: + - A **model-matched preset** has the requested model and can be parsed/planned. + - A **provisionable preset** is a model-matched preset whose run plan has at least one available offer for every job plan. + - Only provisionable presets are submitted automatically. A no-offer preset is shown as context; if `preset_policy: reuse-or-create` and the agent is available, the pipeline falls through to the agent path instead of submitting the doomed preset. With `preset_policy: reuse`, no-offer presets stop with the normal no-offers/no-fleets UX. +- Plan semantics are advisory. `/get_plan` runs the same preset matching code and agent-enabled check that the pipeline will run, but the pipeline re-evaluates after submit because fleets/presets/agent availability may change. This mirrors run apply's UX without introducing endpoint in-place update in v1. +- `src/dstack/_internal/server/routers/endpoints.py`: `project_router = APIRouter(prefix="/api/project/{project_name}/endpoints", tags=["endpoints"], responses=get_base_api_additional_responses())` with `/list`, `/get`, `/get_plan`, `/create`, `/stop`. **Permission: `ProjectMember()`** (volumes precedent — endpoints are project workloads like runs, not admin infra like gateways). `/create` and `/stop` take `pipeline_hinter: Annotated[PipelineHinterProtocol, Depends(get_pipeline_hinter)]`. Responses via `CustomORJSONResponse`. +- Register in `server/app.py::register_routes` next to volumes (~:257). +- API client: `src/dstack/api/server/_endpoints.py` `EndpointsAPIClient(APIClientGroup)` (`list/get/get_plan/create/stop`, parse with `parse_obj_as(Endpoint.__response__, ...)`) + `endpoints` property on `APIClient` (`api/server/__init__.py`). +- No endpoint `/apply_plan` in v1: submit remains `/create`, and changed existing endpoints are handled by CLI-requested stop/override. The CLI requests endpoint stop and polls until the endpoint is terminal; the next `/create` resets the same endpoint row to `SUBMITTED` with the new config. In-place update/rolling redeploy is Later. + +### 5.3 CLI + +- `src/dstack/_internal/cli/services/configurators/endpoint.py`: `EndpointConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator)` with `TYPE = ApplyConfigurationType.ENDPOINT`. Model on `VolumeConfigurator`: + - `apply_configuration`: resolve env sentinels (`apply_env_vars`), call `api.client.endpoints.get_plan(...)`, print the endpoint plan, confirm, then `api.client.endpoints.create(...)`; on name collision: if the existing endpoint has the same config, print no changes and exit unless `--force`; if the config changed, show that the endpoint and its backing service will be stopped and replaced, then require confirmation (or `-y`) before calling `endpoints.stop`, polling until the endpoint becomes terminal, and creating a fresh endpoint. The CLI must not stop backing runs directly. This mirrors run apply's "stop and override" v1 behavior, not an in-place update; in-place update/rolling redeploy is Later (§12). + - Plan output: keep it close to run apply. Print one stable properties table: project, user, configuration path, type, spot policy, max price, preset policy, and preset name only when a provisionable preset was selected. Do not show model, action, endpoint name as a separate row, resources as a separate property, backing service name, agent internals, or a separate "Provisioning" section. If a provisionable preset matched, print offers underneath using the run-style offers table; that offers table is where the concrete resource information belongs. If only no-offer presets matched, print one short message below the table; with `reuse-or-create` and an available agent, continue to the agent path. + - TODO: For `preset_policy: create` or agent fallback, `dstack apply` may later show initial candidate offers the agent can consider. These offers are advisory only: they must not be presented as selected endpoint resources or final hardware because the agent may still choose a different working service shape after investigation. + - Non-detached apply should **imitate attached progress without using attach**: poll `endpoints.get` for status and stream the endpoint's own progress log stream. Do not reserve local ports, open SSH tunnels, or call `run.attach`. Backing service stdout/stderr remains available through `dstack logs `. + - Detached apply (`-d`) submits and exits with the normal `submitted, detaching...` message. Follow-up commands (`dstack endpoint list`, `dstack endpoint get --json`, and `dstack endpoint logs `) are documented/help text, not extra foreground hints. + - `delete_configuration`: unsupported for endpoints. `dstack delete -f endpoint.dstack.yml` should fail clearly and point to `dstack endpoint stop `. + - Provisioning can take tens of minutes (model pulls, agent runs), so `-d` should work exactly as with runs. Keep foreground output concise; document follow-up commands in help/docs. +- Register in `apply_configurators_mapping` (`cli/services/configurators/__init__.py:27-49`). `dstack apply` and `dstack apply -h endpoint` then work with no further CLI edits; `dstack delete -f endpoint.dstack.yml` is explicitly rejected by `EndpointConfigurator.delete_configuration`. +- `src/dstack/_internal/cli/commands/endpoint.py`: `EndpointCommand` (`dstack endpoint list` / `get --json` / `logs` / `stop` / `preset list|get --json|delete`); register in `cli/main.py`. +- Nice property to document: preset-backed endpoints use a readable endpoint-derived service run name (normally `-serving`). Agent-backed endpoints may use a different concise run name chosen during investigation; once linked, backing service logs are available by the linked run name. + +### 5.4 Endpoint logs + +Support endpoint logs in v1, but only via detached/server-side log polling: + +- Add `dstack endpoint logs `. Top-level `dstack logs` remains run-only so same-name endpoint/run collisions do not silently show the wrong stream. Endpoint logs are concise agent/pipeline progress events, not backing service stdout/stderr. +- Do not introduce `dstack attach` for endpoints in v1. Endpoint apply/logs must use configured log storage (`FileLogStorage`, CloudWatch, GCP Logging, Fluent Bit/Elasticsearch, etc.) via the existing server log service. Backing service logs remain normal run logs and are requested with the service run name. +- The agent should emit major progress events to the endpoint log stream in real time. Detailed command output/transcripts stay in the endpoint workspace/debug trace so server logs and endpoint logs do not fill with huge YAML or CLI output. +- Tests: `dstack endpoint logs` reads the endpoint stream and does not fall through to the backing service run; top-level `dstack logs` remains run-only; non-detached endpoint apply does not call `run.attach`. + +--- + +## 6. Background processing: `EndpointPipeline` + +New file `src/dstack/_internal/server/background/pipeline_tasks/endpoints.py`, cloned from `volumes.py` (the cleanest single-model pipeline), registered in `PipelineManager.__init__` (`background/pipeline_tasks/__init__.py:35-48`). + +### 6.1 Pipeline wiring + +- `EndpointPipelineItem(PipelineItem)` + `{status}`; `EndpointPipeline(Pipeline[EndpointPipelineItem])`, `hint_fetch_model_name -> EndpointModel.__name__`. +- Tuning: `workers_num=4` (each in-flight agent run occupies a worker for minutes — this is the per-replica concurrency bound for paid agent sessions), `min_processing_interval=10s`, `lock_timeout=30s`, `heartbeat_trigger=15s`. The `Heartbeater` extends the lease indefinitely during a long `process()` call (precedent: `instances/cloud_provisioning.py`), so minutes-long agent steps are safe while the replica lives. +- `EndpointFetcher.fetch` — copy the volume fetch query shape, with the endpoint status filter and a longer effective interval for RUNNING rows (precedent: `InstanceFetcher`, `instances/__init__.py:190-211`, uses `min_processing_interval * 2` for steady-state statuses; we use a larger multiplier via an explicit condition): + - `WHERE status IN (SUBMITTED, CLAUDING, PROVISIONING, STOPPING) OR (status == RUNNING AND last_processed_at <= now - ENDPOINT_RUNNING_CHECK_INTERVAL)` + - `AND (last_processed_at <= now - min_processing_interval OR last_processed_at == created_at)` + - `AND (lock_expires_at IS NULL OR lock_expires_at < now)` + - `AND (lock_owner IS NULL OR lock_owner == EndpointPipeline.__name__)` + - `ORDER BY last_processed_at ASC LIMIT :limit FOR UPDATE SKIP LOCKED (key_share)` + - claim: one `lock_token` per batch, `lock_expires_at = now + lock_timeout`, `lock_owner = EndpointPipeline.__name__`; commit. +- `EndpointWorker.process`: refetch by `(id, lock_token)` with joinedloads — **must include `EndpointModel.project → ProjectModel.backends`** (in addition to `user` and `service_run`): the preset-matching path reaches `get_project_backends(project)`, which iterates the lazy `project.backends` relationship and raises `MissingGreenlet` under async SQLAlchemy if not eager-loaded (established pattern: `background/pipeline_tasks/volumes.py:237`, `gateways.py:232`). On token mismatch `log_lock_token_mismatch`; dispatch by state; build `_EndpointUpdateMap(ItemUpdateMap)` (adds `status`, `status_message`, `service_run_id`, `provisioning_method`); final write via `update(EndpointModel).where(id==..., lock_token==...).values(**update_map).returning(id)` after `set_processed_update_map_fields` + `set_unlock_update_map_fields` + `resolve_now_placeholders`; 0 rows ⇒ `log_lock_token_changed_after_processing` and abandon. Emit status-change events in the same transaction. Instrument with `@sentry_utils.instrument_pipeline_task(...)`. + +**Transaction rule (critical):** run submission/termination (`submit_run`, `stop_runs`, `delete_runs`) commit their own sessions and take their own advisory locks. The worker must call them through a **fresh `get_session_ctx()`**, never inside its own fetch/update session. Interleaving is safe because the endpoint row itself stays lease-locked. + +**Interim token-guarded updates:** if the agent path needs to persist progress before a long agent call, use an interim `UPDATE ... WHERE id AND lock_token` (+ commit). This is mechanically consistent with the framework (it leaves the lock columns untouched, so the heartbeater and the final write are unaffected; precedent for mid-process commits: related-row lock claims in `fleets.py` / `jobs_submitted.py`) — but the handler must then make sure the final update map doesn't overwrite interim values with stale ones. + +### 6.2 State machine + +``` +SUBMITTED ──(preset selected; deterministic service run name is taken)──► FAILED "run name '' is taken by an existing run" +SUBMITTED ──(preset matched; run submitted)───────► PROVISIONING (method=preset:) +SUBMITTED ──(no preset, agent on)─────────────────► CLAUDING (method=agent) +SUBMITTED ──(preset_policy=reuse; no preset)──────► FAILED "No matching endpoint presets found." +SUBMITTED ──(effective policy=create; agent off)──► FAILED "No matching endpoint presets found. Preset + policy create requires the server agent, + but DSTACK_AGENT_ANTHROPIC_API_KEY is not set." +CLAUDING ─(agent reports verified service run; run not ready yet)──► CLAUDING (service_run_id linked) +CLAUDING ─(linked run RUNNING + replica registered + model URL)────► RUNNING +PROVISIONING ─(preset run RUNNING + replica registered + model URL)► RUNNING +PROVISIONING ─(run finished/soft-deleted)──────────────────────────► FAILED +RUNNING ─(run gone/failed/deleted)─────────────────────────────────► FAILED (stop backing run) # later: agent retry policy +any ─(stop requested)──────────────────────────────────────────────► STOPPING → stop linked backing run → STOPPED +``` + +V1 does **not** automatically retry failed provisioning attempts. A backing run that finishes or fails before readiness makes the endpoint FAILED. + +#### `_process_submitted_endpoint` +1. Parse `EndpointConfiguration`; ask endpoint planning code (`planning.py::find_matching_preset_plan(...)`) to list presets through `EndpointPresetService`, build candidate run specs, and call the run planner (§7). +2. If a provisionable preset matches: check the run name in that selected plan. A non-terminal run conflict that is not the linked `service_run_id` fails with a clear message; terminal conflicts are left to normal run submission, which can recycle them like service apply. Then submit the matching plan in a fresh session as the endpoint's creator user through a small internal helper that mirrors the run router (`refresh_ssh_key` if needed, `get_plan`, then `apply_plan`/`submit_run` with the same policy-applied spec), and return `{status: PROVISIONING, provisioning_method: f"preset:{preset.name}", service_run_id: run.id}`. `register_service` runs synchronously during submission and can raise (`FORBID_SERVICES_WITHOUT_GATEWAY`, referenced gateway missing) — catch `ServerClientError` ⇒ FAILED with the message. +3. Else if agent enabled (`settings.AGENT_ANTHROPIC_API_KEY` set and the real agent runtime is available): return `{status: CLAUDING, provisioning_method: "agent"}` — the agent runs in the CLAUDING handler (keeps SUBMITTED processing fast and makes long agent work visible in CLI/API). Do not block this path on the preset path's deterministic service-run-name convention; the agent may use its own concise candidate/final run names, and the server links the final run by ID after validating project/user/config. +4. Else FAILED (message above; if the key is set but the runtime is unavailable, say the server agent implementation/runtime is unavailable; normal server installs and Docker images must include runtime dependencies once the implementation lands). + +#### `_process_provisioning_endpoint` — reconcile-first +1. **If `service_run_id` set**, load the run (treating `run.deleted == True` as "run gone"): + - run `RUNNING`, ≥1 replica job with `registered == True`, and `ServiceSpec.model.base_url` exists → if `provisioning_method == "agent"`, save a sanitized preset from the backing run (§7.4), then `RUNNING`. This deliberately relies on the existing service probe/registration path rather than adding a second endpoint probe. + - run in `finished_statuses()` or soft-deleted → FAILED with the run error/status. Automatic retry is Later. + - run `TERMINATING` → no-op (wait). + - run still starting (SUBMITTED/PROVISIONING/PULLING) → no-op; stay PROVISIONING. +3. **CLAUDING / no run yet + method=agent** — the long step: + a. Needed hardening: before launching or relaunching the agent, reconcile the endpoint workspace, existing final-report artifacts, endpoint submission rows, and live candidate/final runs. Without this, a server restart can start a fresh agent process for an endpoint whose previous agent process had already submitted work but had not linked `service_run_id` yet. Cleanup of endpoint-submitted runs must use explicit `EndpointRunSubmissionModel` rows, never name matching. + b. Run `AgentService.provision_endpoint(...)` (§8) inside the worker — heartbeater keeps the lease alive while the agent subprocess runs. V0 still needs stop-time cancellation and restart-safe recovery: stop can mark the endpoint `STOPPING`, but the worker must learn to stop/abort the live agent process instead of waiting for it to exit naturally. Do not add a generic endpoint provisioning timeout here; real agent-session budgets belong inside the harness once token/cost/spend semantics are defined. + c. On structured success `{run_id}`/linked service run → set `service_run_id` and keep status `CLAUDING` until the linked service run also satisfies normal dstack service readiness. Once ready, save the learned preset and mark the endpoint `RUNNING`. On failure/abort → FAILED with the agent's summary in `status_message` + stop only the linked service run; cleanup of non-final candidate runs must use explicit endpoint submission records, not name matching. +4. **No run yet + method=preset**: reachable only if the preset run vanished before the endpoint recorded its linked service run — FAILED. Automatic retry is Later. + +#### `_process_running_endpoint` +Runs at the slower RUNNING cadence (`ENDPOINT_RUNNING_CHECK_INTERVAL`, default 60s): +1. Run liveness: backing run missing/soft-deleted/`finished_statuses()`/`TERMINATING` ⇒ FAILED "Backing service run is " (v1; re-provisioning is a Later "agent retry policy" item). This also catches out-of-band `dstack stop`/`delete` of the run by users. +2. No generic server-side endpoint model probe in v1. The backing service owns probes and registration for lifecycle readiness. In the agent path, the agent must make a final model request and include the result in its final report before the server links the candidate. + +**Rule: every transition to FAILED issues a one-shot `stop_runs(abort=False)` for a still non-terminal backing run** (RunPipeline completes the termination asynchronously; no waiting needed since FAILED endpoints aren't re-fetched). Stopping a FAILED endpoint later is a no-op unless a linked run still needs cleanup. + +#### `_process_stopping_endpoint` +Two-phase, server-side, and intentionally simple in v1: (1) backing run present and not finished → `stop_runs(abort=False)` once, then no-op wait on subsequent iterations while the normal RunPipeline terminates the run. (2) run finished (or absent/deleted) → write `{status: STOPPED}`, keep the endpoint row visible in history, and emit "Endpoint stopped". Forced abort/escalation for stuck stopping is Later, not required for v1. + +### 6.3 Crash recovery summary + +Replica dies mid-step ⇒ heartbeats stop ⇒ lease expires (≤ ~30s) ⇒ another replica re-fetches the row. V1 handlers are idempotent around the linked `service_run_id`; preset-run-name conflicts are treated as conflicts, not ownership. `EndpointRunSubmissionModel` preserves endpoint-submitted run history so later agent attempts can recover without relying on names. Agent session budgeting is Later and should be explicit to the agent harness, not a generic endpoint provisioning timeout. Automatic retry is Later. + +### 6.4 Endpoint-level model probes (deferred) + +Do **not** add a generic endpoint probe loop in v1. Service configurations already own probes, and `JobModel.registered` is the service readiness signal after those probes pass. Adding a second endpoint probe loop duplicates service behavior, introduces token/base-URL/proxy/network ambiguity, and makes endpoint lifecycle less conventional for dstack. + +Later, server-side endpoint retry/hardening may add its own explicit verification before re-saving a learned preset or marking an endpoint running again. If/when that is added, prefer reusing the existing in-process probe machinery (`scheduled_tasks/probes.py::_execute_probe` and `get_service_replica_client`) rather than probing through the public service URL. + +### 6.5 Run identity & linkage (decision) + +- **Backing service run name**: the preset path uses `get_endpoint_serving_run_name(endpoint.name)` (`-serving` when it fits, otherwise the endpoint name) because the server submits that service itself. The agent path may use any concise, unique run name. In both paths, `EndpointModel.service_run_id` is the authoritative link for readiness, URL derivation, logs, and v1 cleanup. Name lookup is not an ownership proof and must not be used to adopt or destroy a run. +- **Endpoint-created run ownership:** `EndpointModel.service_run_id` points to the current/latest service run. `EndpointRunSubmissionModel` records ordered endpoint-submitted run history. Keep `RunModel` endpoint-agnostic. +- Consequences for conservative v1: a non-terminal preset-path service run name not linked by `service_run_id` ⇒ endpoint FAILED with a clear message before preset submission. A terminal conflicting run is handled by the existing run submission path. Agent retry policy and cleanup of non-final agent experiment runs are Later/Path B work. + +### 6.6 Endpoint run submissions + +`EndpointModel.service_run_id` is the current/latest service run and the ownership link for cleanup/readiness/log fallback. `EndpointRunSubmissionModel` records ordered run history submitted by the endpoint: + +```python +class EndpointRunSubmissionModel(BaseModel): + endpoint_id: uuid.UUID + run_id: uuid.UUID + submission_num: int + submitted_at: datetime +``` + +Constraints: +- `UNIQUE(run_id)` so a run is recorded for at most one endpoint. +- `UNIQUE(endpoint_id, submission_num)` so submissions are ordered per endpoint. +- Index `endpoint_id` for endpoint history/cleanup lookup. + +Rules: +- `EndpointModel.service_run_id` remains the single current serving run. +- `EndpointRunSubmissionModel` is history/provenance for endpoint-submitted runs, not the serving-run selector. +- Preset-path run-name lookup remains conflict detection for non-terminal runs only. CLI/user confirmation may stop that conflicting run, but the background worker must not auto-adopt or auto-delete it by name. + +Implementation note: +- `runs.apply_plan()` can commit internally, so endpoint code records the submission immediately after a run is accepted and before moving on. If the agent creates multiple prototype/candidate runs inside one call, its submit tool must record each `EndpointRunSubmissionModel` row before the next submission. + +--- + +## 7. Preset subsystem + +Storage and parsing live in `src/dstack/_internal/server/services/endpoints/presets.py`. +Preset matching and run-plan construction live in `src/dstack/_internal/server/services/endpoints/planning.py`. + +### 7.1 What a preset is + +A preset is **one tested endpoint deployment topology on one tested set of replica spec groups**. It is not a generic service recipe that dstack is free to reshape onto arbitrary GPUs, and it is not a bundle of alternative hardware variants. If the same model is tested on L4 and A10G, that is two presets. If one tested service uses multiple replicas or multiple replica groups, that is still one preset. + +In v1 the local file is a small wrapper around: +- `model`: the endpoint model this preset satisfies and the matching key; +- `service`: the serving recipe and service shape (image, commands, port, model mapping, env names, volumes, probes, scaling, replica count/range, or replica groups); +- `replica_spec_groups`: ordered replica groups with one scheduling `resources` value and one `tested_resources` entry per verified running replica. + +The wrapper is deliberate: scheduling requirements and placement evidence are separated from the serving recipe in the preset artifact, then compiled into a normal `ServiceConfiguration` before calling the existing run planner. This keeps the stored preset clear without introducing a new provisioning path. + +Homogeneous service example (`qwen3-32b-vllm-h100x4.dstack.yml`): no explicit `service.replicas` list means there is one implicit replica group (`"0"`). The loader derives the service's top-level `resources` from the group's `resources` before planning/submission. The group's `tested_resources` records where the replicas actually ran when verified. + +```yaml +type: endpoint-preset +model: Qwen/Qwen3-32B +service: + image: vllm/vllm-openai:latest + commands: + - | + vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 \ + --tensor-parallel-size $DSTACK_GPUS_NUM + port: 8000 + model: Qwen/Qwen3-32B + env: + - HF_TOKEN + volumes: + - instance_path: /root/.cache + path: /root/.cache + optional: true +replica_spec_groups: + - name: "0" # optional for the implicit group; stored explicitly when saving + resources: + shm_size: 16GB + gpu: H100:4 + tested_resources: + - cpu: 64 + memory: 512GB + disk: 200GB + gpu: H100:80GB:4 +``` + +Replica-group example: `replica_spec_groups` is sorted in exactly the same order as `service.replicas`, and every group name must match. The loader derives each replica group's service `resources` from the corresponding preset group's `resources`. `tested_resources` records the exact instances that were running when the preset was verified. This supports an existing dstack service shape with different resource specs per group without making v1 responsible for inventing advanced serving architectures. + +```yaml +type: endpoint-preset +model: Qwen/Qwen3-32B +service: + port: 8000 + model: Qwen/Qwen3-32B + replicas: + - name: router + count: 1 + image: ghcr.io/example/router:latest + commands: + - python router.py + - name: worker + count: 2 + image: vllm/vllm-openai:latest + commands: + - vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 +replica_spec_groups: + - name: router + resources: + cpu: 4 + tested_resources: + - cpu: 8 + memory: 16GB + disk: 100GB + gpu: 0 + - name: worker + resources: + shm_size: 16GB + gpu: H100:4 + tested_resources: + - cpu: 64 + memory: 512GB + disk: 200GB + gpu: H100:80GB:4 + - cpu: 64 + memory: 512GB + disk: 200GB + gpu: H100:80GB:4 +``` + +V1 constraints: +- `service` must not contain `name`, `ProfileParams`, top-level `resources`, or replica-group `resources`. Endpoint `ProfileParams` constrain placement/fleets/pricing; they do not change the preset's tested replica spec groups. +- `replica_spec_groups` order is part of the contract. If `service.replicas` is a list, `replica_spec_groups[i].name == service.replicas[i].name` for every group. If there is no replica-group list, the preset has exactly one implicit group (`"0"`), analogous to the default service replica group. +- Each replica group has one `resources` value used for offer matching and service submission. `tested_resources` may contain multiple exact entries, one per verified running replica. If the service needs different scheduling requirements, model those as separate replica groups. +- A preset has exactly one tested replica-spec topology. Hardware alternatives, benchmarked variants, and curated multi-choice registries are Later. + +### 7.2 Interfaces + +```python +class EndpointPreset(CoreModel): + name: str # file stem + model: str # matching key + replica_spec_groups: list[EndpointPresetReplicaSpecGroup] # ordered like service replica groups + """Sorted to match `configuration.replica_groups`; group "0" is the implicit group.""" + configuration: ServiceConfiguration # compiled service config, not the raw file + +class EndpointPresetReplicaSpecGroup(CoreModel): + """Ordered to match `ServiceConfiguration.replica_groups`; "0" is the implicit group.""" + name: str # replica group name; "0" for implicit group + resources: ResourcesSpec # scheduling requirements for each replica in the group + tested_resources: list[ResourcesSpec] # exact verified resources for running replicas + +class EndpointPresetService(ABC): # storage abstraction only — local dir first, S3/git later + @abstractmethod + async def list_presets(self, project_name: str) -> list[EndpointPreset]: ... + +class LocalDirEndpointPresetService(EndpointPresetService): + """Reads *.yml/*.yaml from settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets". + Re-reads per call; file IO via run_async. + Invalid files are logged and skipped, never fatal.""" + +class EndpointPlanReplicaSpecGroup(CoreModel): + name: str + resources: ResourcesSpec + tested_resources: list[ResourcesSpec] + +class EndpointPlanJobOffers(CoreModel): + replica_group: str + offers: list[InstanceOfferWithAvailability] # capped by max_offers + total_offers: int + max_price: Optional[float] + +@dataclass(frozen=True) +class EndpointPresetPlan: # planning.py, not presets.py + preset: EndpointPreset + run_plan: RunPlan +``` + +Module-level `get_endpoint_preset_service()` returning the configured implementation (test-injectable). + +### 7.3 Matching & submission + +This is endpoint planning/orchestration logic, not `EndpointPresetService` storage logic. + +`planning.py::find_matching_preset_plan(session, project, user, endpoint_name, endpoint_conf, preset_service=None) -> Optional[EndpointPresetPlan]`: +1. Candidates: presets whose top-level `model` equals `endpoint_conf.model` case-insensitively, in sorted file-name order. If there are no candidates, return `None` without refreshing the user's SSH key or calling the run planner. +2. For each candidate, build the merged service config (below), wrap in a `RunSpec` (repo fields `None` → virtual repo; `RunSpec.profile` is Optional), and call `runs.get_plan(session, project, user, run_spec, max_offers=1)`. Use the endpoint's effective `creation_policy`; if omitted, keep the normal run default `reuse-or-create` so elastic fleets may provision new instances. If the user explicitly sets `creation_policy: reuse`, matching becomes existing-instances-only. Match ⇔ every `job_plan` has ≥1 offer with `offer.availability.is_available()`. + - Before plan/submission, mirror the run router's behavior: if the creator user has no `ssh_public_key`, call `users.refresh_ssh_key(...)` rather than failing the endpoint. This keeps endpoint apply consistent with `dstack apply` for services. + - **Wrap each candidate in `try/except ServerClientError` → skip**: `get_plan` validates the merged config and can raise for a bad preset; one bad preset must not abort matching. + - Cost note: the planner enumerates cloud backend offers per candidate cloud fleet (`plan.py::get_job_plans` → `find_optimal_fleet_with_offers`) — each candidate evaluation can take seconds. This is acceptable in `/get_plan` and the pipeline only because matching filters by model before planning and uses `max_offers=1`; keep the local preset set small and do not broaden matching into a registry scan in v1. + - Known under-check, accepted for v1 (matching is advisory): `runs.get_plan` produces one representative `JobPlan` per replica group (`replica_num=0`), while the preset may record multiple tested resources per group. V1 verifies each group has at least one available matching offer for its `resources` and leaves full cardinality/capacity checks to the run scheduler. Capacity-aware matching over every tested replica is Later. +3. First match wins. + +`EndpointPresetPlan` contains the selected `EndpointPreset` and the `RunPlan` computed from the merged `RunSpec`. The caller must submit that same plan/spec path rather than rebuilding independently, mirroring the run apply split between `runs.get_plan` and `runs.apply_plan`. + +Merged service config (preset → endpoint overrides): +- start from the preset's compiled `ServiceConfiguration` (`service` plus resources derived from the ordered `replica_spec_groups` at the proper service/replica-group level); +- `name = get_endpoint_serving_run_name(endpoint.name)` (the backing service run name); +- merge `endpoint.env` **over** preset env (`Env.update`); endpoint env arrives fully resolved (sentinels rejected at create); +- copy every non-`None` `ProfileParams` field from `EndpointConfiguration` onto the service config (both inherit `ProfileParams`, so this is a field-loop like `RunSpec._merged_profile`, `core/models/runs.py:590-607`); +- do **not** merge or override resources from the endpoint: endpoints do not expose resources in v1, and the preset's replica group `resources` are the tested scheduling requirements; +- do not force `creation_policy = reuse`: the normal default `reuse-or-create` is intentional because existing dstack fleets may be elastic and allowed to provision new instances for submitted services/jobs. + +How this runs without the agent: the local preset loader injects `replica_spec_groups[0].resources` into top-level `service.resources` for the implicit group, or injects `replica_spec_groups[i].resources` into `service.replicas[i].resources` for explicit replica groups. It also leaves `service.replicas`/`scaling` intact, so the normal service planner/submission path decides how many replicas to start and where to place them. `len(replica_spec_groups[i].tested_resources)` records the verified running replica count at save time; it is evidence and future inspect/JSON input, not a replacement for service `replicas`/autoscaling. + +Submission: `RunSpec(run_name=get_endpoint_serving_run_name(endpoint.name), configuration=merged, ssh_key_pub=None)` as the creator user, using the same policy/SSH-key behavior as the run router (`refresh_ssh_key` if needed; do not let `get_plan` and the final run submission see different specs). If validation still fails, surface that as endpoint FAILED. + +Fleet-drift note: matching is advisory — a fleet can become busy between match and provisioning. No hard endpoint→fleet binding. + +### 7.4 Saving agent-proven presets (v1) + +When an agent-created backing service becomes `RUNNING` through the normal service readiness path, save a sanitized `endpoint-preset` wrapper back into the endpoint project's local preset directory (`/projects//presets`). This makes the second endpoint for the same model in the same project take the deterministic preset path instead of paying the agent again. + +Implementation details: +- Extend `EndpointPresetService` with project-scoped operations such as `save_preset(project_name, preset, comments) -> EndpointPreset`. The local implementation writes a `type: endpoint-preset` YAML file under `settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets"` using an atomic temp-file rename. File names should be stable and collision-resistant, e.g. `-.dstack.yml`; do not overwrite hand-authored presets. +- Source of truth is the backing run's stored `RunSpec.configuration` plus the latest successful job submissions after the service has reached `RUNNING`, not the agent's final text. Sanitize before writing: top-level `model` comes from the verified service model; `replica_spec_groups[*].resources` is copied from the verified service configuration's per-group scheduling requirements; `replica_spec_groups[*].tested_resources` are built from jobs grouped by `JobSpec.replica_group` in `ServiceConfiguration.replica_groups` order, deriving each exact instance `ResourcesSpec` from `JobRuntimeData.offer` when present or the backing `InstanceModel.offer` as fallback. The `service` section preserves the serving recipe/shape fields (`image`, `commands`, `port`, `model`, `volumes`, `replicas`, probes, scaling) but clears `type`, `name`, all resource fields, and `ProfileParams`. Merge env as names only for endpoint-provided keys and redact secret-looking values (`TOKEN`, `KEY`, `SECRET`, `PASSWORD`) to name-only entries. Never write resolved secret values from `EndpointConfiguration.env` to disk. +- Preserve provenance as YAML comments at the top (comments do not affect preset parsing): endpoint name/id, run id, dstack version, timestamp, agent model, job ids that produced the replica spec groups, and recipe source URLs from the agent's structured final report. Do not add a metadata object in v1. +- Failure to save the preset logs a warning and emits an event, but it does **not** fail an already-running endpoint. Tests should cover successful write, duplicate-name avoidance, invalid destination permissions, and secret redaction. + +--- + +## 8. Agent subsystem + +`src/dstack/_internal/server/services/endpoints/agent/`. + +**Current planning note:** this section has been reset to the CLI-first Path B: the agent uses the real `dstack` CLI in a bounded workspace and performs final functional verification. The server owns endpoint state, run linking, service-readiness bookkeeping, and preset saving; it must not treat its own readiness checks as proof that the requested model works. + +### 8.1 Abstraction + +```python +@dataclass(frozen=True) +class AgentProvisioningResult: + run_id: Optional[uuid.UUID] = None + run_name: Optional[str] = None + error: Optional[str] = None + +class AgentService(ABC): + @abstractmethod + def is_enabled(self) -> bool: ... + @abstractmethod + def get_plan(self) -> AgentPlan: ... + @abstractmethod + async def provision_endpoint( + self, + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, + ) -> AgentProvisioningResult: ... +``` + +`get_agent_service()` returns the real Claude agent service when enabled, else a disabled stub. Fully injectable for tests (pytest runs with `--disable-socket`; nothing in the test suite may touch the live agent runtime or external APIs). The pipeline requires a successful agent verification report plus the backing `run_id`/`run_name` or an error. Provenance details flow to preset saving/logging only after the server also confirms normal service readiness. + +### 8.2 `ClaudeAgentService` — real agent runtime + +**v1 must use a real Claude agent runtime, not a raw LLM API loop.** The previous raw `anthropic` Messages prototype was removed because it recreated a primitive tool loop instead of giving the agent a real shell/filesystem/code workflow. The current v0 implementation invokes the Claude Code executable as a subprocess from an endpoint workspace. If Claude Code proves unsuitable after live runs, choose another real agent runtime; only fall back to building a custom loop after an explicit design review. + +What this means for v1: +- *Runtime hardening through live runs*: keep the Claude Code subprocess path intentionally simple, then harden it based on real endpoint attempts. It must operate non-interactively, use the real `dstack` CLI, create/edit files, inspect logs/status, preserve artifacts, emit a final machine-readable report, avoid duplicate live agent processes after restarts, and support endpoint stop aborts. +- *CLI-first dstack operations*: planning, submission, status inspection, log reading, and stopping are done through the same `dstack` CLI a user would run: `dstack apply`, `dstack ps`, `dstack run get --json`, `dstack logs`, and `dstack stop`. Do not build duplicate custom tools named `submit_service`, `get_run_status`, `get_run_logs`, etc. +- *Termination contract*: the agent returns a structured final report with success/failure, final candidate run name/id, final service YAML/content, recipe sources, and verification summary. The agent is responsible for final functional verification. The server does not trust the report blindly; it validates the claimed run identity and project/user ownership before linking the run, then waits for normal dstack service readiness before activating the endpoint. The readiness gate is bookkeeping, not a server-side claim that the model works. +- *Environment hygiene*: the agent runtime receives only the credentials it needs to authenticate to Claude plus the isolated dstack CLI config for the target project. The command/workspace environment remains scrubbed: do not expose server DB credentials, encryption keys, cloud backend credentials, or unrelated server env vars. +- *Abortability*: endpoint stop must be able to stop or ask the agent runtime to stop. This is not complete in the v0 subprocess runtime yet: the worker can hold the endpoint lease while Claude runs, and stop-time cancellation/restart-safe recovery must be hardened before production use. + +Packaging requirement: agent runtime dependencies must be installed automatically by normal server deployment paths. The server runtime must not depend on `uv` being available. The Claude Agent SDK currently depends on pydantic v2 while dstack still runs pydantic v1, so v0 invokes the Claude Code executable directly instead of importing the SDK into the server process. Docker release and staging images copy the bundled Claude Code binary into `/usr/local/bin/claude` at build time. `DSTACK_AGENT_CLAUDE_PATH` may point the server at a specific Claude Code executable; when it is unset, `ClaudeAgentService` resolves `claude` from `PATH`. For non-Docker installs, automatic non-Docker packaging for that executable remains a release-blocking follow-up before the agent path can be considered production-ready. + +Packaging gate before enabling `ClaudeAgentService`: +- The server process stays dependency-compatible with pydantic v1 and does not require `uv` at runtime. +- Server Docker release and staging images include the Claude Code executable without an operator shelling into the container to install it manually. +- The availability check used by `get_agent_service()` validates the API key and required `claude` executable, and reports a clear server-install problem if packaging regresses. +- Tests or CI checks cover at least the Python dependency metadata and runtime availability probe; Docker verification is part of the manual e2e runbook if it is too heavy for unit CI. + +### 8.3 CLI-first harness + +The important product work is the harness, not the HTTP loop. The agent should behave like a careful dstack developer using the CLI, with strong context and boundaries: +1. understand the requested model and endpoint constraints; +2. gather grounded recipe + hardware evidence from credible sources; +3. inspect the project context and available fleet/offer envelope through CLI/status output; +4. draft the smallest service YAML that should work; +5. preview with `dstack apply -f ` before spending GPU time; +6. submit detached with `dstack apply -f -y -d` only when the plan is acceptable within the endpoint/profile/budget envelope; +7. debug with `dstack ps`, `dstack run get --json`, and `dstack logs`; +8. stop bad candidates with `dstack stop -y` and let normal run lifecycle handle termination/deletion; +9. finish only after the final service is RUNNING, has a registered replica, exposes a model URL, and the agent has made a final model request proving the requested model is actually served. + +This is intentionally **not** a new server-side mini API for dstack. The harness should provide: +- an endpoint-scoped working directory for service YAMLs, notes, and command transcripts; +- the real `dstack` binary from the running installation; +- a CLI config/context that targets the correct server, project, and endpoint creator user; +- endpoint constraints: model, env names, ProfileParams, preset policy, and allowed resource/spend envelope; +- recipe grounding guidance and optionally preloaded recipe/context snippets, not a required `find_model_recipes` tool; +- command logging and a `should_abort()` check between command iterations so deletion can interrupt long agent work; +- a structured final-report schema. + +The preset path uses a server-owned deterministic run name from `get_endpoint_serving_run_name(endpoint.name)` because the server submits that service itself. The agent path does not force that naming convention. Claude may choose concise, unique candidate and final run names; the server validates and links the final service by reported run ID, then records it in `EndpointModel.service_run_id` and `EndpointRunSubmissionModel`. + +Dev-environment and task prototyping are allowed in the v0 agent prompt when they are the fastest way to reduce uncertainty about an image, launch command, model download, or hardware choice. For v1, advanced P/D disaggregation, multi-service routers/workers, load benchmarking, and autoscaling tuning are Later unless needed to make the requested model serve at all. + +### 8.4 Prompt, recipe grounding & vendored context + +Runtime context layout: + +- `resources/system_prompt.md` — endpoint-specific mission and protocol only: use the real CLI, load `/dstack` and `/dstack-prototyping`, honor endpoint constraints, emit concise progress events, verify the final model API request, and return the structured final report. +- repo-root `skills/dstack/SKILL.md` — CLI/config source of truth. +- repo-root `skills/dstack-prototyping/SKILL.md` — reusable research-to-working-workload skill: source order, model-serving recipe selection, hardware fit, dev-environment/task/service experiment choice, failure classification, final service cleanup, and model API verification. + +`pyproject.toml` force-includes repo-root `skills/**` into wheels/sdists. The workspace setup locates that packaged `skills` directory and copies only `dstack` and `dstack-prototyping` into `.claude/skills` for each endpoint-agent run. The prompt explicitly names `/dstack` and `/dstack-prototyping`; do not rely on an operator's user-level Claude/Codex skills. + +`dstack-prototyping` should answer both "how do I experiment on dstack?" and "what should I try deploying for this model/framework/hardware?" It must stay generic to dstack workload prototyping and must not mention endpoint statuses, preset saving, endpoint DB rows, or Claude-specific implementation details. Endpoint-specific requirements stay in `system_prompt.md`. + +Recipe grounding should be source-oriented, not a static recipe encyclopedia. The agent should prefer current primary sources such as model cards, vLLM/SGLang docs, dstack docs/CLI help, and its own command/log evidence. Advanced posts such as Wafer GLM-on-AMD and LMSYS agent-assisted SGLang development are directional for future harness work, not v1 requirements. + +The prompt interpolates endpoint env keys (names only), ProfileParams constraints, project context, and effective agent budget. Recipe grounding is discovered by the agent through allowed network/command facilities and recorded in the final report; do not model it as a required `find_model_recipes` function in v1. + +### 8.5 Execution, cost & limits + +- Runs inside the pipeline worker (§6.2) unless the chosen runtime forces a detached process model. Agent execution must not block the event loop; use async subprocess/process supervision or a small dedicated executor with bounded concurrency, not the shared 128-thread default executor. +- Between agent/runtime steps, or through runtime cancellation hooks, the service checks `should_abort()` (cheap SELECT of stop intent + `lock_token` sanity) and exits early on stop. +- Do **not** add generic endpoint provisioning timeouts or unused turn/attempt counters. The real agent loop should introduce budgets only where they are enforced. v0 uses the endpoint configuration's `max_agent_budget` value, falling back to `DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD`, and passes the effective value to Claude Code's `--max-budget-usd` as the hard per-agent-process cost cap. Once retries/resumes or multiple Claude processes per endpoint provisioning are added, dstack must persist and sum agent spend per endpoint provisioning attempt before starting the next process, so the total endpoint provisioning budget cannot be bypassed by restarts. +- Model via `DSTACK_AGENT_ANTHROPIC_MODEL` if the selected runtime supports explicit model selection (default: `claude-opus-4-8`; the default agent should bias toward the strongest Anthropic model because endpoint provisioning is an expensive, tool-heavy deployment investigation). +- Concurrency bound = pipeline `workers_num` per replica (4). +- Observability: log each command + final summary at INFO with the endpoint id; store command transcripts under the endpoint workspace or log service for debugging; store the final agent summary in `status_message` on failure; log provider usage for token-budget accounting; store `recipe_sources` in the saved preset comments on success. Cost accounting/events → Later. + +--- + +## 9. Settings, packaging, docs + +New in `src/dstack/_internal/server/settings.py` (documented in `mkdocs/docs/reference/env.md`, as the module docstring requires): + +| env var | constant | default | notes | +|---|---|---|---| +| `DSTACK_AGENT_ANTHROPIC_API_KEY` | `AGENT_ANTHROPIC_API_KEY` | `None` | as specified by the requirement (agent-scoped, hence no `_SERVER_`); presence ⇒ agent enabled | +| `DSTACK_AGENT_CLAUDE_PATH` | `AGENT_CLAUDE_PATH` | `None` | optional path to the Claude Code executable; falls back to resolving `claude` from `PATH` | +| `DSTACK_AGENT_ANTHROPIC_MODEL` | `AGENT_ANTHROPIC_MODEL` | `claude-opus-4-8` | override only if the operator intentionally wants a cheaper/faster model | +| `DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD` | `AGENT_ANTHROPIC_MAX_BUDGET` | `None` | optional server default for endpoint agent spend; overridden by endpoint `max_agent_budget`; future retry/resume support must sum spend across processes per endpoint provisioning attempt | + +Plain module constants in `settings.py`, not env-configurable in v1: `SERVER_PROJECTS_DIR_PATH = /projects`, `ENDPOINT_RUNNING_CHECK_INTERVAL = 60s`. + +Packaging: the selected real agent runtime dependency/binary is installed automatically without contaminating the server Python environment and without requiring `uv` at runtime. For Docker this means copying the bundled Claude Code executable into the image at build time. For non-Docker server installs, automatic packaging of the `claude` executable must be solved before the agent path is production-ready. Treat this as a release blocker for enabling the Claude agent path, not as optional docs. + +Docs: `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._internal.core.models.endpoints.EndpointConfiguration` (processed by `scripts/docs/gen_schema_reference.py`), added to `mkdocs.yml` nav and the `reference/dstack.yml.md` index; env vars in `reference/env.md`; a short concepts page ("Endpoints — experimental") once the feature works end-to-end. + +--- + +## 10. Implementation milestones (each ≈ one PR) + +**M1 — resource skeleton (no processing).** `core/models/endpoints.py` + registration in `configurations.py`; `EndpointModel` + migration + events wiring; `EndpointPlan` models with the `none` provisioning branch; `services/endpoints/__init__.py` (create/stop/list/get/get_plan shell, status switch, events); schemas + router + `app.py`; `_endpoints.py` API client group; CLI configurator + `EndpointCommand`. Endpoints can be planned/created and sit SUBMITTED forever. Tests: router CRUD, name uniqueness/reapply semantics, configurator parse, sentinel rejection, plan output for no preset/no agent. +**M2 — pipeline & lifecycle.** `background/pipeline_tasks/endpoints.py` + registration; the full state machine of §6.2 with `AgentService`/`EndpointPresetService` as interfaces with disabled/empty defaults (so: SUBMITTED→FAILED "nothing configured", preset-path run-name conflict detection, RUNNING backing-run liveness through `service_run_id`, simple two-phase server-side stop of the linked service run, and FAILED teardown rule). Tests: fetcher query (sqlite+postgres, lock claims, RUNNING cadence, STOPPING handling), every worker transition with fakes, conflict detection, linked-run cleanup. Do not test name-based adoption as ownership. +**M3 — presets + plan offers.** `presets.py` (`EndpointPresetService`, `LocalDirEndpointPresetService`, project-scoped storage/parsing only under `/projects//presets`), `endpoint-preset` wrapper parsing (`service` recipe + ordered tested `replica_spec_groups`), `planning.py` matching via `get_plan` using the endpoint's effective `creation_policy` (default `reuse-or-create`, so elastic fleets are allowed), per-candidate `ServerClientError`/unresolved-env skip, config merge, submission path, `EndpointPlan.provisioning_plan=type:"preset"` only for provisionable presets with tested replica spec groups and offer summary, save-preset interface + sanitizer. Tests: matching unit tests with testing factories (project/fleet/instance), project isolation, merge semantics, invalid preset files skipped, bad-preset-skip, no-offer preset falls through to agent when policy allows, plan prints selected preset/offers/replica spec groups, secret redaction, atomic write/no-overwrite. +**M4 — agent + endpoint logs.** Build on `service_run_id` + `EndpointRunSubmissionModel` so only the latest run serves while submission history is preserved. Continue hardening `ClaudeAgentService` around the v0 Claude Code subprocess runtime: workspace/process handling, scrubbed environment, structured final report artifact, vendored context, `EndpointPlan.provisioning_plan=type:"agent"`, agent settings, automatically installed runtime dependencies, `should_abort`/process cancellation, successful-agent preset save on `RUNNING`, `dstack endpoint logs`, and non-detached endpoint apply following server-side logs/status without attach. Tests should cover real runtime integration boundaries actually used by `ClaudeAgentService`, final-report parsing, runtime availability packaging checks for the `server`/`all` extras, endpoint logs command behavior, apply does not call attach, fake-runtime service integration, and FakeAgentService pipeline integration; mocked network/CLI where possible. Manual e2e: local `uv` server install plus server Docker image both reach the same runtime availability state with only `DSTACK_AGENT_ANTHROPIC_API_KEY` configured. +**M5 — docs & polish.** Reference page, env.md, concepts page, `dstack endpoint` help texts, endpoint plan rendering polish, example presets in docs (not shipped as defaults — §11 Q7), manual e2e runbook (local server + real fleet + one preset + one agent deploy). + +--- + +## 11. Key open questions — with recommended answers + +1. **Run naming & history.** Q: how to link endpoint↔run and what happens on retries? **A:** preset-backed services use `-serving` when valid because the server submits them. Agent-backed services are linked by reported `run_id`; the agent chooses concise unique run names. `service_run_id` is authoritative for the serving run; name lookup is conflict detection only for preset submission. `EndpointRunSubmissionModel(endpoint_id, run_id, submission_num, submitted_at)` records endpoint-submitted run history without coupling `RunModel` to endpoints. Agent retry policy → Later. +2. **Who owns the backing run?** Q: which UserModel submits it (no system user exists)? **A:** the endpoint's creator (`user_id` on the row) — correct attribution in events/quotas; mirror the run router by refreshing the user's server-managed SSH key before `get_plan`/submission if missing. A first-class service account → Later. +3. **Where does the agent execute?** Q: inside the pipeline worker vs a detached task with its own heartbeat bookkeeping? **A:** v0 runs inside the worker `process()` under the Heartbeater lease, with `workers_num=4` as the concurrency bound. This is enough to test the real loop, but stop-time cancellation and restart-safe recovery are not done: `stop_endpoints` can mark the endpoint `STOPPING` without acquiring the lock, but the live Claude subprocess still needs an explicit abort/supervision path. +4. **Which Claude runtime?** **A: v0 uses Claude Code headless/subprocess after rejecting the raw `anthropic` Messages loop.** Continue validating non-interactive server execution, cancellation, workspace/env isolation, final artifact extraction, restart behavior, and packaging. If Claude Code proves unsuitable after real deployments, choose another real agent runtime before adding more harness complexity. +5. **Permissions.** Q: `ProjectMember` (volumes) or `ProjectAdmin` (gateways)? **A:** `ProjectMember` — endpoints are project workloads that consume the same quota surface as runs, not shared admin infrastructure. +6. **Preset match semantics.** Q: what does "matches the existing fleets" mean concretely? **A:** A model-matched preset is only **provisionable** when `get_plan` on the merged config returns ≥1 available offer for every job plan using the endpoint's effective `creation_policy`. Default `reuse-or-create` may return offers from elastic fleets that can create new instances; explicit `reuse` restricts to currently existing instances. Only provisionable presets are submitted automatically. If presets match the model but no offers are available, `preset_policy: reuse` stops with no-offers/no-fleets, while `reuse-or-create` falls through to the agent path if available. Advisory, not a binding; known to under-check multi-replica capacity (§7.3), accepted for v1. +7. **Do we ship built-in presets?** **A:** No — empty preset dir by default; docs show copy-paste presets, and successful agent deployments write local learned presets. Shipping curated presets with the wheel makes dstack responsible for third-party image/version rot on every release; revisit with a versioned preset registry (Later). +8. **Feature flag?** **A:** No `DSTACK_FF_*` flag. The feature is inert unless a preset dir is populated or the agent key is set; a config-type flag would have to gate client-side parsing too, which FFs here don't do. Document as experimental. +9. **Env var naming.** **A:** keep `DSTACK_AGENT_ANTHROPIC_API_KEY` exactly as specified (breaks the `DSTACK_SERVER_*` convention deliberately: it namespaces a future family of `DSTACK_AGENT_*` settings). +10. **Plan/apply surface.** **A:** v1 ships an advisory `get_plan` plus create/stop/list/get/logs/preset. `dstack apply` always shows the endpoint plan before confirmation, but submit remains create-and-background-process; no endpoint `apply_plan` or in-place update until Later. +11. **Endpoint updates.** **A:** stop/override only in v1. Later in-place endpoint updates must reuse existing service-run `get_plan`/`apply_plan` and rolling deployment (`deployment_num`, desired replica counts, service registration/probes). Endpoint DB updates need their own `configuration_version`/deployment guard so a stale worker in a multi-server Postgres deployment cannot mark a newer endpoint config running. + +--- + +## 12. Later (explicitly deferred) + +Carried over from the requirements ("later we…") plus deferrals made above: + +- **Preset sources beyond local dir**: S3, git repo (reuse the `services/templates.py` clone/TTL machinery), HTTP registry; richer provenance metadata (version pins, observed offer, benchmark results), signed/curated preset channels; shipping default presets; capacity-aware matching over every recorded replica spec and exact offer identity. +- **Preset update policy**: allow the agent to update/replace an existing learned preset only as a repair path, not as optimization. The harness must first prove that the old preset is not reproducible under the conditions it claimed to support (for example same model family/recipe constraints and compatible tested resource topology, but repeated normal preset-path attempts fail for reasons attributable to the preset rather than transient capacity/no-offers/user constraints). Only then may it save a replacement or new version, with evidence of the failed reproducibility attempt and the newly verified deployment. V1 learned presets stay append-only/no-overwrite until this policy is designed and tested on real failures. +- **Automatic provisioning retry policy**: retry failed provisioning through SUBMITTED, backoff, retry history retention, and distinct candidate run names if preserving failed candidates becomes important. +- **Agent retry policy for RUNNING endpoints**: instead of FAILED on health-check failure, re-invoke the agent to diagnose/redeploy; backoff policy; `retry` ProfileParams semantics for endpoints. +- **Alternative agent runtime** — revisit if the first real runtime cannot meet server requirements for non-interactive execution, packaging, cancellation, workspace/env isolation, artifact capture, and cost accounting. +- **Other `AgentService` implementations** (OpenAI-compatible, Bedrock/Vertex via the SDK's provider clients, self-hosted). +- **Richer service creation**: deepen `dstack-prototyping` through real endpoint-agent traces; richer attach/SSH automation for dev environments; benchmarking before marking an endpoint running; autoscaling/replicas/gateway/domain decisions; quantization variant selection; multi-node deployments; P/D disaggregation and other multi-component serving topologies; advanced SGLang development/deployment harnesses inspired by the 2026-07-02 LMSYS agent-assisted SGLang workflow. +- **Workload-aware optimization**: benchmark and tune endpoints by workload profile (chatbot, RAG, code generation, long-form generation), measuring TTFT, ITL, throughput, end-to-end latency, quality, and cost/token. Product references such as Modal Auto Endpoints, Makora, and Runpod Overdrive are useful directionally, but v1 must stop at verified functional deployment and reproducible preset provenance. +- **In-place update / plan-apply**: server-side endpoint `get_plan`/`apply_plan`, model or config changes via the existing service-run `apply_plan` + rolling deployment machinery, no stop/recreate for simple changes; endpoint row updates protected by a `configuration_version`/deployment guard for multi-server safety. +- **Deletion escalation**: forced abort / operator-facing stuck-deletion handling if a backing run remains terminating beyond the normal RunPipeline retry window. +- **Plugins**: add an `EndpointSpec` to the `ApplySpec` TypeVar (`dstack/plugins/_models.py:8`) so plugin policies can reason about endpoint-level intent directly. Backing service runs still use normal run policies in v1. +- **Frontend UI**: endpoints page in the server frontend. No frontend work is required in v1. +- **Richer progress surfaces**: agent transcript/event streaming beyond concise status/log-following; structured display of recipe evidence and harness decisions in CLI/UI. +- **Cost & governance**: per-project agent budgets, token/cost accounting per endpoint, API key via the encrypted secrets service (`services/secrets.py`) instead of a server-wide env var, audit events for every agent tool call. +- **Agent hardening**: policy hooks on tool calls, sandboxed execution, structured-output enforcement of the final report beyond the `finish` tool. +- **Vendored recipe snapshots** for air-gapped servers (Apache-2.0 attribution), refresh tooling. +- **Model catalog UX**: `dstack endpoint models` listing known-deployable models from presets + recipes indexes. + +--- + +## 13. Risks & mitigations (summary) + +| risk | mitigation in this plan | +|---|---| +| Replica death mid-agent-run duplicates deployments / spend | `service_run_id` points to the latest run and `EndpointRunSubmissionModel` records accepted endpoint-submitted runs. The heartbeater lease prevents duplicate live workers while the replica lives; after a crash, the next worker reconciles by linked run/submission history. Name lookup is conflict-only (§6.2, 6.3, 6.5) | +| Paid crash loop (pipeline re-fetch + flaky agent) | no automatic retry in v1 + terminal FAILED stops re-fetching; real agent loop must add explicit token/cost/wall-clock budgets before it can spend on experiments | +| Adopting/destroying an unrelated user run with the same name | name lookup is conflict detection only; ownership must come from `service_run_id` or `EndpointRunSubmissionModel` rows, never from name/user/timestamp heuristics (§6.5) | +| One-shot stop+delete of runs always raising | endpoint stop remains two-phase stop → wait-until-finished → stopped. The CLI-agent should use normal `dstack stop -y` for bad candidates and let run lifecycle handle termination (§2.5, §8.3) | +| GPU-run leaks on terminal FAILED | every FAILED transition issues `stop_runs` for a still non-terminal backing run (§6.2) | +| Transaction/lock corruption submitting backing runs from the worker | fresh `get_session_ctx()` for all runs-service calls; endpoint row writes stay token-guarded; any interim progress update leaves lock columns untouched (§6.1) | +| Prompt injection via fetched recipes → server compromise | command execution runs in an endpoint-scoped workspace with a scrubbed environment; the agent uses normal `dstack` CLI rather than server internals; server secrets/DB/cloud credentials and the agent API key are not exposed to commands (§8.2, 8.3) | +| Agent-learned preset writes resolved secrets to disk | preset sanitizer clears `name`, writes endpoint env as names only, redacts secret-looking values, and tests secret redaction (§7.4) | +| Event-loop starvation from long agent IO | supervise the agent runtime with async subprocess/process APIs or a bounded dedicated executor; never the shared default executor; small `workers_num` | +| `RunStatus.RUNNING` ≠ model ready | endpoint `RUNNING` requires a registered running service replica and `ServiceSpec.model.base_url`; endpoint v1 does not add a parallel health probe (§6.4) | +| `MissingGreenlet` on lazy `project.backends` in the worker | refetch joinedloads chain `project → backends` (§6.1) | +| `register_service` raises synchronously during backing run submission (gateway config issues) | caught as submission failure ⇒ FAILED with message; agent prompt includes gateway context | +| Agent deploys a service without `model:` → no model URL to surface | prompt requires `model:`; agent final verification must use the model endpoint, and server readiness validation refuses to mark the endpoint running unless the backing service exposes `ServiceSpec.model.base_url` (§6.2, §8.3) | +| Preset/fleet drift between match and provisioning | matching is advisory; run scheduling is source of truth | +| Agent runtime dependency missing in deployment | selected runtime dependencies are included in normal server install paths and Docker images; release Docker uses `dstack[all]`, staging uses `uv sync --extra all`, and local server installs must not require manual runtime installation (§8.2, 9) | +| Missing migration passes unit tests silently | migration is an explicit M1 review item; postgres-parametrized tests | diff --git a/endpoint-implementation-research/background-loop.md b/endpoint-implementation-research/background-loop.md new file mode 100644 index 0000000000..11e3a8daea --- /dev/null +++ b/endpoint-implementation-research/background-loop.md @@ -0,0 +1,241 @@ +# background-loop + +## Summary +dstack's server background processing has two families, both started from the FastAPI lifespan in src/dstack/_internal/server/app.py (lines 178-184, gated by settings.SERVER_BACKGROUND_PROCESSING_ENABLED): (1) "pipeline tasks" — per-DB-model fetch/worker/heartbeat pipelines (background/pipeline_tasks/base.py) that claim rows with durable DB lock columns (lock_expires_at/lock_token/lock_owner) plus SELECT ... FOR UPDATE SKIP LOCKED, and (2) "scheduled tasks" — APScheduler IntervalTrigger jobs for infrequent, idempotent work. Multi-replica safety comes from the lock columns (not in-memory locks): a replica heartbeats lock_expires_at forward every ~1s while processing, and every mutation is guarded by `WHERE id = :id AND lock_token = :token`; if a replica dies, the lock expires and another replica's fetcher (which selects rows with `lock_expires_at IS NULL OR lock_expires_at < now`, ordered by last_processed_at ASC) takes over with a new token. Long-running work (instance provisioning, 10-55 min) is handled by (a) indefinite heartbeat extension during one `process()` call and (b) chunking across pipeline iterations via a status state machine (PENDING -> PROVISIONING -> per-iteration readiness checks against a deadline). A new `process_endpoints` task should be a new `EndpointPipeline` copied from the VolumePipeline skeleton (the simplest single-model example), registered in PipelineManager, with an EndpointModel using PipelineModelMixin + last_processed_at + a partial fetch index and an alembic migration. + +## Key files +- src/dstack/_internal/server/background/pipeline_tasks/base.py — Pipeline, Fetcher, Worker, Heartbeater, PipelineItem, PipelineModel, ItemUpdateMap, NOW_PLACEHOLDER, set_unlock_update_map_fields, set_processed_update_map_fields, resolve_now_placeholders, log_lock_token_mismatch, log_lock_token_changed_after_processing, log_lock_token_changed_on_reset — The generic pipeline framework every processing task follows; 483 lines, fully read. +- src/dstack/_internal/server/background/pipeline_tasks/__init__.py — PipelineManager, PipelineHinter, get_pipeline_manager, start_pipeline_tasks, PipelineManager.register_pipeline — Where a new EndpointPipeline must be registered (builtin list at lines 35-48). +- src/dstack/_internal/server/background/pipeline_tasks/volumes.py — VolumePipeline, VolumeFetcher.fetch, VolumeWorker.process, _refetch_locked_volume, _apply_process_result, _VolumeUpdateMap, _ProcessResult — Cleanest single-model pipeline; the recommended template for process_endpoints (SUBMITTED->ACTIVE/FAILED via backend calls). +- src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py — RunPipeline, RunFetcher.fetch, RunWorker.process, _lock_related_jobs, _unlock_related_jobs, _reset_run_lock_for_retry — Example of cross-model child-row locking (run locks its jobs with the same lock_token) and per-status dispatch. +- src/dstack/_internal/server/background/pipeline_tasks/fleets.py — FleetPipeline, FleetFetcher.fetch, _lock_fleet_instances_for_processing, _apply_process_result — Second cross-model locking example; also shows exponential consolidation retry delays pattern (_CONSOLIDATION_RETRY_DELAYS). +- src/dstack/_internal/server/services/locking.py — get_locker(dialect_name), ResourceLocker, InMemoryResourceLocker, DummyResourceLocker, advisory_lock_ctx, try_advisory_lock_ctx, string_to_lock_id — In-memory lockset for SQLite, no-op dummy for Postgres; Postgres advisory locks for one-off critical sections. NOT the multi-replica mechanism for row processing. +- src/dstack/_internal/server/models.py — PipelineModelMixin (lines 204-207), RunModel (405), VolumeModel (951), ix_runs_pipeline_fetch_q index (464-472) — lock_expires_at/lock_token/lock_owner mixin + per-model last_processed_at + partial fetch index; EndpointModel must replicate this. +- src/dstack/_internal/server/background/scheduled_tasks/__init__.py — start_scheduled_tasks, get_scheduler, AsyncIOScheduler — APScheduler interval/date jobs; per-replica, no cross-replica dedup; for infrequent idempotent work only. +- src/dstack/_internal/server/services/pipelines.py — PipelineHinterProtocol, get_pipeline_hinter — FastAPI dependency for API handlers to hint fetchers after submitting a row (reduces processing latency). +- src/dstack/_internal/server/app.py — lifespan (start_scheduled_tasks/start_pipeline_tasks at 178-184, shutdown/drain at 206-214, default ThreadPoolExecutor at 123-124) — Startup/shutdown wiring; app.state.pipeline_manager set at line 181. +- src/dstack/_internal/server/db.py — get_db, get_session_ctx, get_session, Database.dialect_name, is_db_sqlite, is_db_postgres, sqlite_commit — Session helpers used by all fetchers/workers; get_session_ctx commits on clean exit; autoflush disabled. +- src/dstack/_internal/server/settings.py — SERVER_BACKGROUND_PROCESSING_ENABLED/DISABLED (47-50), SERVER_EXECUTOR_MAX_WORKERS (52), MAX_OFFERS_TRIED (54) — Only env vars controlling background processing; no per-pipeline tuning env vars exist. +- src/dstack/_internal/server/migrations/versions/2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py — upgrade/downgrade — Most recent 'add a pipeline' migration: adds lock columns + status + last_processed_at, backfills last_processed_at=created_at. Template for endpoints migration. +- src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py — offers loop capped by settings.MAX_OFFERS_TRIED (line 121), compute.create_instance via run_async (172) — Long-running (minutes) process() example, kept alive by heartbeater. +- src/dstack/_internal/server/background/scheduled_tasks/idle_volumes.py — process_idle_volumes — Scheduled-task skeleton: lockset + FOR UPDATE SKIP LOCKED + respects pipeline rows via lock_expires_at IS NULL. +- src/tests/_internal/server/background/pipeline_tasks/test_volumes.py — TestVolumeFetcher, fetcher/worker fixtures, _volume_to_pipeline_item — Test conventions: instantiate Fetcher/Worker with Mock queue/heartbeater, parametrize test_db over ['sqlite','postgres']. + +## Details +All paths relative to /Users/dstack/dstack. Everything below was read from the working tree (branch master, commit 28ea5f86f era). + +## 1. Two background-task families and how they start + +`src/dstack/_internal/server/app.py` lifespan: +- line 123-124: `server_executor = ThreadPoolExecutor(max_workers=settings.SERVER_EXECUTOR_MAX_WORKERS)`; `asyncio.get_running_loop().set_default_executor(server_executor)` — this is the pool `run_async` uses. +- lines 176-184: +```python +if settings.SERVER_BACKGROUND_PROCESSING_ENABLED: + scheduler = start_scheduled_tasks() + pipeline_manager = start_pipeline_tasks() + app.state.pipeline_manager = pipeline_manager +... +PROBES_SCHEDULER.start() # separate AsyncIOScheduler for probes, started unconditionally +``` +- shutdown (206-214): `pipeline_manager.shutdown()` → `scheduler.shutdown()` → `await pipeline_manager.drain()`. + +### pipeline_tasks (`background/pipeline_tasks/__init__.py`) +`start_pipeline_tasks() -> PipelineManager` (line 106) docstring: "Start tasks processed by fetch-workers pipelines based on db + in-memory queues. Suitable for tasks that run frequently and need to lock rows for a long time." + +`PipelineManager.__init__` (lines 31-49) instantiates and registers 12 builtin pipelines, each constructed as `XxxPipeline(pipeline_hinter=self._hinter)`: ComputeGroupPipeline, FleetPipeline, GatewayPipeline, GatewayReplicaPipeline, JobSubmittedPipeline, JobRunningPipeline, JobTerminatingPipeline, InstancePipeline, PlacementGroupPipeline, RunPipeline, ServiceRouterWorkerSyncPipeline, VolumePipeline. Public `register_pipeline(pipeline)` (line 51) appends and registers with the hinter — this is where `EndpointPipeline` gets added. + +`PipelineHinter` (lines 80-93): `_hint_fetch_map: dict[str, list[Pipeline]]` keyed by `pipeline.hint_fetch_model_name` (a `Model.__name__` string, e.g. `"VolumeModel"`); `hint_fetch(model_name)` calls `pipeline.hint_fetch()` on all pipelines registered for that model (all three job pipelines share `JobModel.__name__`). Singleton via `get_pipeline_manager()` (line 99). + +API handlers obtain the hinter via `get_pipeline_hinter(request: Request) -> PipelineHinterProtocol` in `src/dstack/_internal/server/services/pipelines.py:22` (reads `request.app.state.pipeline_manager`, returns a no-op hinter if background processing is disabled). Real usages: `services/volumes.py:317 pipeline_hinter.hint_fetch(VolumeModel.__name__)` after volume create; `services/runs/__init__.py:825-826, 906`; `services/fleets.py:880, 1112-1113`; `services/gateways/__init__.py:291`. Hints are local-replica only (they just set an asyncio.Event on the fetcher). + +### scheduled_tasks (`background/scheduled_tasks/__init__.py`) +`start_scheduled_tasks() -> AsyncIOScheduler` (line 37) docstring: "Start periodic tasks triggered by apscheduler at specific times/intervals. Suitable for tasks that run infrequently and don't need to lock rows for a long time." Jobs (lines 43-60): `init_gateways_in_background` (DateTrigger, once), `preload_offers_catalog` (DateTrigger + IntervalTrigger(minutes=10)), `process_probes` (seconds=3, jitter=1), `collect_metrics` (10s), `delete_metrics` (5m), `delete_events` (7m), `process_gateways_connections` (15s), `process_idle_volumes` (60s, jitter=10), `delete_instance_healthchecks` (5m), plus prometheus pair if `settings.ENABLE_PROMETHEUS_METRICS`. `max_instances=1` on most — note this is per-process only, NOT cross-replica; every replica runs every scheduled task, so they must be idempotent (e.g. `delete_events` is a bare `DELETE WHERE recorded_at < cutoff`, scheduled_tasks/events.py:13-17). + +## 2. The pipeline framework (base.py) — exact skeleton + +`background/pipeline_tasks/base.py`: + +```python +@dataclass +class PipelineItem: # base.py:34 + __tablename__: str + id: uuid.UUID + lock_expires_at: datetime + lock_token: uuid.UUID + prev_lock_expired: bool # set by fetchers, currently consumed nowhere (verified by grep) + +class PipelineModel(Protocol): # base.py:50 — model must have: + __tablename__: str; __mapper__; __table__ + id: Mapped[uuid.UUID] + lock_expires_at: Mapped[Optional[datetime]] + lock_token: Mapped[Optional[uuid.UUID]] + +class Pipeline(Generic[ItemT], ABC): # base.py:67 + def __init__(self, workers_num, queue_lower_limit_factor, queue_upper_limit_factor, + min_processing_interval: timedelta, lock_timeout: timedelta, + heartbeat_trigger: timedelta) -> None: ... + def start(self): ... # creates asyncio tasks: heartbeater.start(), each worker.start(), fetcher.start() + def shutdown(self): ... # stop flags + cancel tasks + async def drain(self): ... + def hint_fetch(self): self._fetcher.hint() + # abstract: hint_fetch_model_name (property str), _heartbeater, _fetcher, _workers +``` +Queue sizing: `_queue_desired_minsize = ceil(workers_num * queue_lower_limit_factor)`, `_queue_maxsize = ceil(workers_num * queue_upper_limit_factor)`; `asyncio.Queue[ItemT](maxsize=_queue_maxsize)`. + +`Fetcher` (base.py:257): loop — if `qsize >= desired_minsize` sleep `queue_check_delay=1.0`; else `items = await self.fetch(limit=maxsize - qsize)`; on empty fetch, wait on `self._fetch_event` with timeout from `_DEFAULT_FETCH_DELAYS = [0.5, 1, 2, 5]` seconds indexed by consecutive-empty count, ±20% jitter, with a 10% random chance of resetting to the minimal delay (base.py:326-337); `hint()` sets the event. Non-empty: `queue.put_nowait(item)` + `heartbeater.track(item)`. Abstract: `async def fetch(self, limit: int) -> list[ItemT]`. + +`Worker` (base.py:340): `__init__(queue, heartbeater, pipeline_hinter: PipelineHinterProtocol)`; loop: `item = await queue.get()`; `await self.process(item)` inside try/except-log; `finally: await self._heartbeater.untrack(item)`. Abstract: `async def process(self, item: ItemT)`. + +`Heartbeater` (base.py:166): `__init__(model_type: type[PipelineModel], lock_timeout, heartbeat_trigger, heartbeat_delay: float = 1.0)`. Every ~1s: for each tracked item, if `lock_expires_at < now` → untrack + warn ("Failed to heartbeat ... in time"); if `lock_expires_at < now + heartbeat_trigger` → include in one bulk `UPDATE model SET lock_expires_at = now + lock_timeout WHERE (id==i.id AND lock_token==i.lock_token) OR ... RETURNING id` (base.py:227-240); items whose token changed are untracked. So a lock is extended indefinitely while the replica is alive and the worker is still processing — this is what makes minutes-long `process()` calls safe. + +Update-map helpers (base.py:379-483): `NOW_PLACEHOLDER` + `resolve_now_placeholders(update_values, now)` so all timestamps in one apply-transaction share the same `now`; `ItemUpdateMap` TypedDict with `lock_expires_at/lock_token/lock_owner/last_processed_at`; `set_unlock_update_map_fields(m)` sets the three lock fields to None; `set_processed_update_map_fields(m, now=NOW_PLACEHOLDER)` sets `last_processed_at`; standard warn-loggers `log_lock_token_mismatch(logger, item, action="process")`, `log_lock_token_changed_after_processing(logger, item, ...)`, `log_lock_token_changed_on_reset(logger)`. + +## 3. Model-side requirements + +`src/dstack/_internal/server/models.py`: +```python +class PipelineModelMixin: # models.py:204 + lock_expires_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + lock_token: Mapped[Optional[uuid.UUID]] = mapped_column(UUIDType(binary=False)) + lock_owner: Mapped[Optional[str]] = mapped_column(String(100)) +``` +Models using it: RunModel(405), ServiceRouterWorkerSyncModel(475), JobModel(506), GatewayModel(600), GatewayComputeModel(656), FleetModel(754), InstanceModel(805), VolumeModel(951), PlacementGroupModel(1011), ComputeGroupModel(1047). + +Each pipeline-processed model also declares its own `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime)` (non-null) and a partial index for the fetch query, e.g. RunModel (models.py:464-472): +```python +Index("ix_runs_pipeline_fetch_q", last_processed_at.asc(), + postgresql_where=status.not_in(RunStatus.finished_statuses()), + sqlite_where=status.not_in(RunStatus.finished_statuses())) +``` +New rows are created with `last_processed_at` = submission time (`services/runs/__init__.py:744 last_processed_at=submitted_at`; `services/volumes.py:307 last_processed_at=now`); fetchers treat `last_processed_at == created_at` (or `== submitted_at` for runs) as "never processed → skip the min-interval gate". Some models additionally have `skip_min_processing_interval: Mapped[bool]` (RunModel:432, JobModel, InstanceModel) which fetchers OR into the interval condition and reset to False on fetch. + +Migration template: `migrations/versions/2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py` — batch_alter_table adds `last_processed_at` (NaiveDateTime), `status`, `status_message`, `lock_expires_at`, `lock_token` (UUIDType(binary=False)), `lock_owner` (String(100)); backfills `last_processed_at = created_at`; then makes columns non-null. Migrations live under `src/dstack/_internal/server/migrations/versions//`. + +## 4. Standard fetch skeleton (verified in VolumeFetcher.fetch, volumes.py:135-196; same shape in runs/fleets/instances) + +```python +volume_lock, _ = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__) +async with volume_lock: # asyncio.Lock on SQLite; DummyAsyncLock (no-op) on Postgres + async with get_session_ctx() as session: + now = get_current_datetime() + res = await session.execute( + select(VolumeModel) + .where( + or_(VolumeModel.status == VolumeStatus.SUBMITTED, VolumeModel.to_be_deleted == True), + VolumeModel.deleted == False, + or_(VolumeModel.last_processed_at <= now - self._min_processing_interval, + VolumeModel.last_processed_at == VolumeModel.created_at), + or_(VolumeModel.lock_expires_at.is_(None), VolumeModel.lock_expires_at < now), + or_(VolumeModel.lock_owner.is_(None), VolumeModel.lock_owner == VolumePipeline.__name__), + ) + .order_by(VolumeModel.last_processed_at.asc()) + .limit(limit) + .with_for_update(skip_locked=True, key_share=True, of=VolumeModel) + .options(load_only(VolumeModel.id, VolumeModel.lock_token, VolumeModel.lock_expires_at, ...))) + volume_models = list(res.scalars().all()) + lock_expires_at = get_current_datetime() + self._lock_timeout + lock_token = uuid.uuid4() # ONE token per fetched batch + for m in volume_models: + prev_lock_expired = m.lock_expires_at is not None + m.lock_expires_at = lock_expires_at; m.lock_token = lock_token + m.lock_owner = VolumePipeline.__name__ + items.append(VolumePipelineItem(__tablename__=VolumeModel.__tablename__, id=m.id, ...)) + await session.commit() +return items +``` +Fetchers are decorated `@sentry_utils.instrument_pipeline_task("VolumeFetcher.fetch")` (workers likewise with `"VolumeWorker.process"`); `instrument_pipeline_task(name)` / `instrument_scheduled_task(f)` live in `server/utils/sentry_utils.py:14-19`. + +## 5. Standard worker skeleton (VolumeWorker.process, volumes.py:212-291) + +1. Refetch with full relationships: `select(VolumeModel).where(VolumeModel.id == item.id, VolumeModel.lock_token == item.lock_token).options(joinedload(...))` → `scalar_one_or_none()`; if None → `log_lock_token_mismatch(logger, item)`; return. +2. Do the actual processing as pure-ish functions returning a `_ProcessResult` containing a TypedDict update map (`_VolumeUpdateMap(ItemUpdateMap)` adds `status`, `status_message`, `volume_provisioning_data`, `deleted`, `deleted_at`). Blocking backend/SDK calls go through `run_async` (`src/dstack/_internal/utils/common.py:49-51`: `await asyncio.get_running_loop().run_in_executor(None, partial(func, *args, **kwargs))` — the default executor is the 128-thread pool from app.py:123). +3. Apply: `set_processed_update_map_fields(update_map)`; `set_unlock_update_map_fields(update_map)`; `resolve_now_placeholders(update_map, now=get_current_datetime())`; then +```python +res = await session.execute( + update(VolumeModel) + .where(VolumeModel.id == volume_model.id, VolumeModel.lock_token == volume_model.lock_token) + .values(**update_map).returning(VolumeModel.id)) +if len(list(res.scalars().all())) == 0: + log_lock_token_changed_after_processing(logger, item); return +``` +plus event emission (`services.events.emit(session, msg, actor=events.SystemActor(), targets=[events.Target.from_model(model)])` and per-domain `emit_*_status_change_event`). + +## 6. Cross-model (child row) locking — runs and fleets + +`RunPipeline._lock_related_jobs` (runs/__init__.py:763-810): under the jobs lockset lock, `select(JobModel).where(JobModel.run_id == item.id, status not in JOB_STATUSES_EXCLUDED_FOR_LOCKING, lock free or expired, lock_owner NULL or == RunPipeline.__name__).order_by(JobModel.id).with_for_update(skip_locked=True, key_share=True, of=JobModel)`; then re-selects ALL current eligible job ids — if the locked set != full set, it gives up: `_reset_run_lock_for_retry` (runs/__init__.py:813-835) which keeps `lock_owner` (so the row stays owned by the pipeline and other subsystems stay away), but sets `lock_expires_at=None, lock_token=None, last_processed_at=now` so the item is retried on a later fetch and the heartbeater can no longer touch it. Children get the parent's `lock_expires_at/lock_token` and `lock_owner=RunPipeline.__name__`. On every apply/noop path, `_unlock_related_jobs` (950-969) nulls the three lock columns `WHERE id IN locked AND lock_token == item.lock_token AND lock_owner == RunPipeline.__name__`. FleetPipeline does the same for InstanceModel (`fleets.py:374-445`, unlock at 762-779), and InstanceFetcher avoids fighting the fleet by requiring `FleetModel.lock_owner IS NULL` for instances in a fleet (instances/__init__.py:212-224). + +Related: run/fleet workers unlock/update parent + children in ONE transaction; job update rows are built with `set_unlock_update_map_fields`/`set_processed_update_map_fields` per row and executed as bulk `await session.execute(update(JobModel), job_update_rows)` (runs/__init__.py:588-589). + +## 7. Locking service — exact API (`src/dstack/_internal/server/services/locking.py`) + +- `get_locker(dialect_name: str) -> ResourceLocker` (line 175): returns module-level `InMemoryResourceLocker` for `"sqlite"`, else `DummyResourceLocker` ("We could use an in-memory locker on Postgres but it can lead to unnecessary lock contention, so we use a dummy locker that does not take any locks."). NOTE: it takes `dialect_name` as a required arg — call sites are all `get_locker(get_db().dialect_name)`. +- `ResourceLocker.get_lockset(namespace: str) -> tuple[LocksetLock, Lockset]` — a guard lock plus a set of locked keys; `lock_ctx(namespace, keys)` acquires all keys (keys must be sorted to avoid deadlock; implemented by `_wait_to_lock_many(lock, locked, keys, delay=0.1)`). +- `string_to_lock_id(s) -> int` (sha256 mod 2**63); `advisory_lock_ctx(bind, dialect_name, resource)` (line 125) — `pg_advisory_lock`/`pg_advisory_unlock`, NO-OP on SQLite, with documented footguns (must release on the same connection; don't commit inside when bind is an AsyncSession); `try_advisory_lock_ctx` (line 156) yields a bool. Used for `migrate()` (db.py:83) and `server_init` (app.py:140). + +**How multi-replica double-processing is actually prevented for pipelines**: not by the locker. It's (a) the durable `lock_expires_at`/`lock_token`/`lock_owner` columns claimed in the fetch transaction, (b) `WITH FOR UPDATE SKIP LOCKED` making concurrent Postgres fetch transactions skip each other's rows mid-claim, and (c) every subsequent write being conditioned on `lock_token`. On SQLite (single replica by definition) `with_for_update` is a no-op and the in-memory lockset lock serializes fetchers/scheduled-task claimers within the process instead. + +## 8. Replica death / stale lock recovery + +If a replica dies mid-processing: heartbeats stop → `lock_expires_at` (20-40s in the future) passes → any replica's fetcher matches the row again via `or_(lock_expires_at.is_(None), lock_expires_at < now)` and issues a NEW `lock_token`; `prev_lock_expired=True` is recorded on the item (currently informational only). If the dead replica's worker somehow finishes later, its `UPDATE ... WHERE lock_token == old_token` matches 0 rows and is logged, not applied. There is no separate reaper/janitor; recovery latency ≈ remaining lock_timeout. `ORDER BY last_processed_at ASC` guarantees stalest-first pickup. If `process()` raises, the Worker logs and unt racks; the row simply stays locked until expiry (retry latency ≈ lock_timeout). + +## 9. Long-running operations (minutes+) + +Two mechanisms, both exemplified by instances: +1. **Heartbeat-extended single call**: `InstancePipeline` PENDING processing (instances/cloud_provisioning.py) loops over offers, each `compute.create_instance` executed via `run_async` (line 172), loop capped by `settings.MAX_OFFERS_TRIED` (line 121, default 25, "Limit number of offers tried to prevent long-running processing in case all offers fail"). The Heartbeater keeps extending the row lock every ~1s check for as long as needed. `JobSubmittedPipeline` similarly calls `run_async(compute...)` at jobs_submitted.py:1816, 2287, 2309. +2. **Chunked state machine across iterations**: after `create_instance` returns, the worker only writes `status=PROVISIONING` + `job_provisioning_data` + `started_at=NOW_PLACEHOLDER` and unlocks (cloud_provisioning.py:208-231). Each subsequent pipeline iteration (~min_processing_interval) re-picks the PROVISIONING row and checks readiness against `get_provisioning_deadline` (instances/check.py:200-204, 297-301), with per-backend timeouts from `get_provisioning_timeout(backend_type, instance_type_name)` in `background/pipeline_tasks/common.py:6` (10 min default, up to 55 min for Vultr bare metal). **This is the pattern to copy for an endpoint LLM-agent flow: persist per-phase status (e.g. SUBMITTED → PROVISIONING/DEPLOYING → health-checking) so each `process()` is resumable if a replica dies, rather than one hours-long process() call.** + +## 10. Pipeline tuning defaults (constructor kwargs; no env vars) + +| Pipeline | workers_num | min_processing_interval | lock_timeout | heartbeat_trigger | +|---|---|---|---|---| +| RunPipeline (runs/__init__.py:56) | 10 | 5s (x2 for non SUBMITTED/TERMINATING) | 30s | 15s | +| JobSubmittedPipeline (jobs_submitted.py:158) | 40 | 4s | 40s | 20s | +| JobRunningPipeline (jobs_running.py:138) | 20 | 5s | 30s | 15s | +| JobTerminatingPipeline (jobs_terminating.py:91) | 20 | 2s | 30s | 15s | +| InstancePipeline (instances/__init__.py:81) | 20 | 7s (x2 for idle/busy) | 30s | 15s | +| FleetPipeline (fleets.py:69) | 10 | 15s | 20s | 10s | +| VolumePipeline (volumes.py:61) | 10 | 15s | 30s | 15s | +| GatewayPipeline (gateways.py:62) | 10 | 15s | 30s | 15s | +| GatewayReplicaPipeline (gateway_replicas.py:56) | 10 | 15s | 30s | 15s | +| ComputeGroupPipeline (compute_groups.py:49) | 10 | 15s | 30s | 15s | +| PlacementGroupPipeline (placement_groups.py:46) | 10 | 15s | 30s | 15s | +| ServiceRouterWorkerSyncPipeline (service_router_worker_sync.py:54) | 8 | 5s | 25s | 10s | +All use queue_lower_limit_factor=0.5, queue_upper_limit_factor=2.0. + +## 11. Settings / env vars (server/settings.py) + +- `DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED` → `SERVER_BACKGROUND_PROCESSING_DISABLED/ENABLED` (lines 47-50) — the only kill switch; disables BOTH scheduled and pipeline tasks (app.py:178). +- `DSTACK_SERVER_EXECUTOR_MAX_WORKERS` (line 52, default 128) — thread pool for `run_async`. +- `DSTACK_SERVER_MAX_OFFERS_TRIED` (line 54, default 25). +- `DSTACK_DB_POOL_SIZE` (44, default 20) / `DSTACK_DB_MAX_OVERFLOW` (45, default 20). +- `DSTACK_ENABLE_PROMETHEUS_METRICS` gates the prometheus scheduled tasks. +There are NO env vars for per-pipeline workers/intervals/lock timeouts — constructor defaults only. + +## 12. Scheduled-task skeleton (if endpoints ever needed one) + +`process_idle_volumes` (scheduled_tasks/idle_volumes.py:24-63): decorated `@sentry_utils.instrument_scheduled_task`; gets `(lock, lockset) = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__)`; under `async with lock:` selects candidate ids with `.where(..., VolumeModel.lock_expires_at.is_(None), VolumeModel.id.not_in(lockset)).limit(10).with_for_update(skip_locked=True, key_share=True)` and adds ids to the lockset; processes; `finally: lockset.difference_update(volume_ids)`. Note it defers real work to the pipeline by setting `to_be_deleted=True`. The `lock_expires_at.is_(None)` check is how a scheduled task avoids touching rows a pipeline currently holds. + +## 13. Precise recipe for a `process_endpoints` pipeline task + +1. **Model**: `class EndpointModel(PipelineModelMixin, BaseModel)` in `src/dstack/_internal/server/models.py` with `id UUIDType(binary=False) pk default uuid4`, `created_at`, `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime)` (init to created_at on insert), `status` (use `EnumAsString(EndpointStatus, 100)`), `status_message`, `deleted: Mapped[bool]`, FKs to projects/users, and `__table_args__ = (Index("ix_endpoints_pipeline_fetch_q", last_processed_at.asc(), postgresql_where=, sqlite_where=),)`. +2. **Migration**: alembic revision under `src/dstack/_internal/server/migrations/versions/2026/`, modeled on `857d8fa7fcc5` (or a plain create_table since it's a new table). +3. **Pipeline**: `src/dstack/_internal/server/background/pipeline_tasks/endpoints.py` with `EndpointPipelineItem(PipelineItem)` (+ `status` field), `EndpointPipeline(Pipeline[EndpointPipelineItem])` (copy VolumePipeline verbatim: __init__ wires `Heartbeater(model_type=EndpointModel, ...)`, `EndpointFetcher`, N× `EndpointWorker`; `hint_fetch_model_name -> EndpointModel.__name__`; expose `_heartbeater/_fetcher/_workers` via private-name properties), `EndpointFetcher.fetch` copying the section-4 query (status filter e.g. `status.in_([SUBMITTED, PROVISIONING, ...])`, `lock_owner IS NULL OR == EndpointPipeline.__name__`), `EndpointWorker.process` doing refetch-by-token → per-status dispatch → apply-with-token pattern. Suggested tuning for agent-driven provisioning: workers_num ~10, min_processing_interval 5-15s, lock_timeout 30s, heartbeat_trigger 15s (heartbeater makes multi-minute steps safe, but prefer chunking into statuses per section 9.2). +4. **Register** in `PipelineManager.__init__` builtin list (`background/pipeline_tasks/__init__.py:35-48`). +5. **Submit-side hint**: in the endpoint-create service function, accept `pipeline_hinter: PipelineHinterProtocol` (injected in the router via `Depends(get_pipeline_hinter)`, as `services/volumes.py` does) and call `pipeline_hinter.hint_fetch(EndpointModel.__name__)` after commit. +6. **Sub-resource**: if the endpoint pipeline must mutate its RunModel/service, follow the `_lock_related_jobs` pattern (claim child rows with same lock_token + lock_owner=EndpointPipeline.__name__, reset-own-lock-for-retry if children unavailable) — or better, avoid cross-pipeline row writes and interact with runs via the `services/runs` submit/terminate service functions the way API handlers do (cross-pipeline direct writes are only done for the benign `skip_min_processing_interval` flag, jobs_running.py:990-999). +7. **Tests**: `src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py`, mirroring `test_volumes.py`: fixtures constructing `EndpointFetcher(queue=asyncio.Queue(), queue_desired_minsize=1, ..., heartbeater=Mock())` and `EndpointWorker(queue=Mock(), heartbeater=Mock(), pipeline_hinter=Mock())`, `@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)`, helpers from `dstack._internal.server.testing.common`. + +## Gotchas +1. STALE KNOWLEDGE TRAP: the old `process_runs.py`-style `@sentry_utils.instrument_background_task` + apscheduler-per-N-seconds processing modules do NOT exist anymore; runs/jobs/fleets/instances/volumes/gateways are all fetch/worker Pipelines under background/pipeline_tasks/. Also `get_locker()` now REQUIRES a dialect_name arg: `get_locker(get_db().dialect_name)`. +2. The in-memory lockset locker is NOT the multi-replica mechanism — on Postgres it's a no-op DummyResourceLocker by design. Cross-replica exclusion = lock_expires_at/lock_token/lock_owner columns + FOR UPDATE SKIP LOCKED in the fetch + token-guarded UPDATEs. Any plan that says "acquire the locker to be replica-safe" is wrong. +3. Every write after fetch MUST carry `WHERE lock_token == item.lock_token` and every terminal apply MUST both unlock (set_unlock_update_map_fields) and bump last_processed_at (set_processed_update_map_fields) in the SAME UPDATE; forgetting last_processed_at causes hot-looping, forgetting unlock stalls the row for lock_timeout. +4. `min_processing_interval` gating uses `last_processed_at == created_at` (or `== submitted_at` for runs) to fast-path brand-new rows — so the row-creation code must set last_processed_at equal to created_at, not leave it NULL (column is non-nullable). +5. APScheduler `max_instances=1` and DateTrigger init tasks are per-replica; scheduled tasks run concurrently on every replica and must be idempotent. Don't put endpoint provisioning there — use a pipeline (that's exactly what the two docstrings at pipeline_tasks/__init__.py:107-109 and scheduled_tasks/__init__.py:38-41 distinguish). +6. `hint_fetch` only wakes the local replica's fetcher; without it processing still happens within a few seconds via the polling fetch delays (max ~5s + jitter), so it's an optimization, not a correctness requirement. +7. Worker `process()` exceptions are swallowed+logged by Worker.start(); the row then stays locked until lock_expires_at passes (~lock_timeout retry latency). If a replica dies mid-process, work is re-picked from scratch after lock expiry — so the endpoint agent/deploy flow must be idempotent or persisted as a status state machine (see instances PENDING→PROVISIONING→deadline-checked pattern) rather than a single monolithic process() that runs for many minutes; the heartbeater technically allows unbounded process() duration, but crash-recovery restarts the whole step. +8. `run_async` uses the loop's default executor (ThreadPoolExecutor, 128 threads, app.py:123); long blocking calls (LLM API calls, SSH) should go through it or a native-async client, or they'll block heartbeats for the entire replica. +9. DB sessions: `get_session_ctx()` commits on clean exit, `expire_on_commit=False`, `autoflush=False` — mutations made on ORM objects during fetch are only persisted because fetch explicitly commits; in workers prefer explicit `update()` statements over ORM attribute mutation. +10. `prev_lock_expired` on PipelineItem is set by all fetchers but consumed nowhere — don't build logic assuming it does something. +11. `lock_owner` is a pipeline-name namespace: fetchers only take rows where lock_owner is NULL or their own class name, and the reset-for-retry path deliberately KEEPS lock_owner while nulling lock_expires_at/lock_token. If two subsystems (e.g. an endpoints pipeline and the run pipeline) can lock the same table, both must honor this filter. +12. Datetimes are stored via NaiveDateTime columns while `get_current_datetime()` returns tz-aware UTC — copy existing comparison patterns verbatim rather than mixing aware/naive by hand. +13. Background processing can be disabled entirely (DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED, used in tests); services must not assume the pipeline manager exists — `get_pipeline_hinter` already handles this with a no-op hinter. diff --git a/endpoint-implementation-research/claude-agent-sdk.md b/endpoint-implementation-research/claude-agent-sdk.md new file mode 100644 index 0000000000..5e39a4a6c7 --- /dev/null +++ b/endpoint-implementation-research/claude-agent-sdk.md @@ -0,0 +1,407 @@ +# claude-agent-sdk + +## Summary +Comparison of Claude Agent SDK (Python) vs. plain anthropic SDK for embedding an autonomous deployment agent in dstack. **Recommend Agent SDK (v0.2.110, Python 3.10+) for first implementation**: it bundles the Claude Code CLI binary (not Node.js), provides built-in autonomous tool execution with permission control modes, handles the agentic loop automatically, and supports skills loading convention. The plain anthropic SDK (v0.116.0, Python 3.9+) is lighter-weight but requires manual tool-loop implementation. Both authenticate via ANTHROPIC_API_KEY. Agent SDK deployment footprint: bundled binary CLI + Python runtime ~300MB per platform. Defer plain SDK approach until Agent SDK proves insufficient for the autonomous deployment use case. + +## Key files +- claude-agent-sdk (PyPI: pip install claude-agent-sdk) — query(), ClaudeSDKClient, ClaudeAgentOptions, @tool, create_sdk_mcp_server(), HookMatcher, AgentDefinition, AsyncAnthropic pattern — Official Python Agent SDK. Version 0.2.110 released June 24, 2026. Requires Python 3.10+. Bundles Claude Code CLI binary (not Node.js; packed as platform-specific wheels for macOS ARM64/x86-64, Linux, Windows). GitHub: anthropics/claude-agent-sdk-python. Bundling mechanism: CLI is included in wheel distribution; used by default via stdio; custom path override supported via ClaudeAgentOptions(cli_path=...). +- anthropic (PyPI: pip install anthropic) — Anthropic, AsyncAnthropic, messages.create(), messages.stream(), @beta_tool, tool_runner, messages.count_tokens(), messages.batches, ToolUseBlock, ToolResultBlock — Official Anthropic Python SDK. Version 0.116.0 released July 2, 2026. Requires Python 3.9+. Pure Python, no external binaries. GitHub: anthropics/anthropic-sdk-python. Includes optional extras: [bedrock], [vertex], [aws] for cloud integrations; [aiohttp] for better async concurrency. +- https://code.claude.com/docs/en/agent-sdk/overview — Built-in tools, hook callbacks, permission_mode values, mcp_servers dict config, agents dict, allowed_tools/disallowed_tools lists — Official Agent SDK documentation. Covers query()/ClaudeSDKClient interfaces, built-in tools (Read, Write, Edit, Bash, Monitor, Glob, Grep, WebSearch, WebFetch, AskUserQuestion), hooks lifecycle (PreToolUse, PostToolUse, Stop, SessionStart, SessionEnd, UserPromptSubmit), subagents, MCP integration, permission model, sessions with resume(). Skills convention documented: .claude/skills/*/SKILL.md auto-loaded from working directory and ~/.claude/. + +## Details +## A) Claude Agent SDK (Python) + +### Package & Installation +- **Name**: `claude-agent-sdk` +- **Version**: 0.2.110 (June 24, 2026) +- **Python**: 3.10+ +- **Installation**: `pip install claude-agent-sdk` +- **Bundling**: Bundles Claude Code CLI **as a binary** (not Node.js). Platform-specific wheels (macOS ARM64/x86-64, Linux, Windows). No separate Node.js installation or subprocess complexity. + +### Core APIs + +**Simple interface:** +```python +from claude_agent_sdk import query, ClaudeAgentOptions + +async for message in query( + prompt="Deploy model X as dstack service", + options=ClaudeAgentOptions(allowed_tools=["Bash", "Read", "Write"]) +): + print(message) +``` + +**Interactive sessions:** +```python +from claude_agent_sdk import ClaudeSDKClient + +async with ClaudeSDKClient(options=options) as client: + await client.query(prompt) + async for msg in client.receive_response(): + print(msg) +``` + +### Custom Tool Definition +Via `@tool` decorator + in-process MCP server (no subprocess overhead): +```python +from claude_agent_sdk import tool, create_sdk_mcp_server + +@tool("verify_endpoint", "Verify endpoint reachability", {"url": str}) +async def verify_endpoint(args): + # Direct Python execution + return {"content": [{"type": "text", "text": f"OK: {args['url']}"}]} + +server = create_sdk_mcp_server( + name="dstack-tools", + version="1.0.0", + tools=[verify_endpoint] +) + +options = ClaudeAgentOptions( + mcp_servers={"tools": server}, + allowed_tools=["mcp__tools__verify_endpoint"] # Pre-approve +) +``` + +### Built-In Tools (Pre-Integrated) +Read, Write, Edit, Bash, Monitor, Glob, Grep, WebSearch, WebFetch, AskUserQuestion. All available without configuration. + +### Permission Model (for Autonomous Headless Operation) +```python +options = ClaudeAgentOptions( + allowed_tools=["Read", "Write", "Bash"], # Auto-approve list + disallowed_tools=["DangerousTool"], # Block list + permission_mode='acceptEdits' # Auto-accept file edits +) +``` +Evaluation order: `allowed_tools` → `disallowed_tools` → `permission_mode` fallback. No interactive approval prompts when tools pre-approved. **Can run fully autonomous in headless Docker environment.** + +### Hooks (for Observability & Control) +```python +async def log_exec(input_data, tool_use_id, context): + action = input_data.get("tool_input", {}) + audit_log.write(f"{tool_use_id}: {action}\n") + return {} + +options = ClaudeAgentOptions( + hooks={ + "PostToolUse": [ + HookMatcher(matcher="Bash", hooks=[log_exec]) + ] + } +) +``` + +### Authentication & Model Selection +- **API Key**: `ANTHROPIC_API_KEY` environment variable (standard) +- **Models**: Supports Claude Opus 4.8, Sonnet 4.6, Haiku 4.5, Fable 5 (June 2026 launch), Mythos 5 (trusted access) +- **Model config**: Set via options or defaults to latest available + +### Skills Loading +SDK supports `.claude/` filesystem convention: +- `.claude/skills/*/SKILL.md` – Auto-loaded skill definitions +- `.claude/CLAUDE.md` – Project memory/context +- `setting_sources` parameter to restrict which sources load + +**Not explicitly documented**: whether SDK has programmatic API to load skills from custom directories (e.g., `setting_sources` list-based config). This may require filesystem symlinks or copying skills into `.claude/`. + +### Sessions & Context Persistence +```python +# Capture session ID on init +session_id = message.data["session_id"] + +# Resume later with full context +async for msg in query(prompt="...", options=ClaudeAgentOptions(resume=session_id)): + ... +``` + +### Deployment Footprint +- **Per-platform binary**: ~100–150 MB (CLI binary alone) +- **SDK library**: ~50 MB +- **Python runtime**: depends on container base +- **No Node.js required**: Binary CLI, not JavaScript runtime +- **Subprocess management**: Minimal; SDK handles stdio communication with bundled binary +- **Concurrent sessions**: Each async query spawns a CLI process; manage concurrency via asyncio limits + +## B) Plain anthropic Python SDK + +### Package & Installation +- **Name**: `anthropic` +- **Version**: 0.116.0 (July 2, 2026) +- **Python**: 3.9+ +- **Installation**: `pip install anthropic` +- **Size**: ~10 MB, pure Python + +### Core API (Manual Tool-Use Loop) + +```python +from anthropic import Anthropic + +client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) + +messages = [{"role": "user", "content": "Deploy model X"}] +tools = [{"name": "bash", "description": "...", "input_schema": {...}}] + +while True: + response = client.messages.create( + model="claude-opus-4-8", + max_tokens=4096, + tools=tools, + messages=messages + ) + + if response.stop_reason == "tool_use": + # Manual loop: find tool calls, execute, collect results + for block in response.content: + if block.type == "tool_use": + result = execute_tool(block.name, block.input) + messages.append({"role": "assistant", "content": response.content}) + messages.append({ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": block.id, + "content": result + }] + }) + break + else: + break # stop_reason == "end_turn" + +print(response.content[-1].text) +``` + +### Tool Definition (Helper Decorator) +```python +from anthropic import beta_tool + +@beta_tool +def bash_exec(command: str) -> str: + """Execute bash command. Args: command (str): bash to run""" + return subprocess.check_output(command, shell=True, text=True) + +@beta_tool +def verify_endpoint(url: str) -> str: + """Verify endpoint. Args: url (str): endpoint URL""" + import requests + return "OK" if requests.head(url).ok else "FAIL" +``` + +The `@beta_tool` decorator auto-generates the tool schema from function signature and docstring. **No built-in permission control**; you control tool availability by including/excluding from the `tools` list. + +### Tool Runner Helper (Simplifies Loop) +```python +from anthropic import beta_tool + +runner = client.beta.messages.tool_runner( + model="claude-opus-4-8", + max_tokens=4096, + tools=[bash_exec, verify_endpoint], + messages=[{"role": "user", "content": "Deploy model X"}] +) + +for message in runner: + if hasattr(message, "content"): + print(message.content) +``` +Still requires you to implement tool execution; the runner just collects the loop boilerplate. **Not equivalent to Agent SDK's autonomous execution.** + +### Streaming +```python +with client.messages.stream( + model="claude-opus-4-8", + max_tokens=4096, + tools=tools, + messages=messages +) as stream: + for text in stream.text_stream: + print(text, end="", flush=True) + final = stream.get_final_message() +``` + +### Web Search & Web Fetch Tools +Both available as built-in tools (since April 2026): +- `web_search_20260318` (dynamic filtering variant) – $10 per 1,000 searches + standard token costs +- `web_fetch_20250305` – Included in standard requests +Models supporting web tools: Opus 4.7+, Sonnet 4.6+, Opus 4.6+, Opus 4.5+, Haiku 4.5+, Sonnet 4.5+ + +### Prompt Caching +```python +response = client.messages.create( + model="claude-opus-4-8", + max_tokens=4096, + system=[ + { + "type": "text", + "text": "You are a deployment agent...", + "cache_control": {"type": "ephemeral"} + } + ], + messages=[...] +) +``` +Cached input costs 90% less. Useful for long system prompts. Works across multiple requests within 5-minute window. + +### Token Counting +```python +count = client.messages.count_tokens( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + tools=tools +) +print(count.input_tokens, count.output_tokens) + +# Post-request usage +response = client.messages.create(...) +print(response.usage.input_tokens, response.usage.output_tokens) +``` + +### Structured Outputs +```python +response = client.messages.create( + model="claude-opus-4-8", + max_tokens=4096, + messages=[...], + structured_output={"schema": {...}, "strict": True} +) +# Returns validated JSON matching schema +``` + +### Authentication & Model Selection +- **API Key**: `ANTHROPIC_API_KEY` environment variable +- **Models**: Same lineup as Agent SDK (Opus 4.8, Sonnet 4.6, Haiku 4.5, Fable 5, Mythos 5) +- **Cloud provider integrations**: Bedrock, Vertex AI, Foundry, Claude Platform on AWS (via `AnthropicBedrock`, `AnthropicVertex`, etc.) + +### Deployment Footprint +- **Pure Python**: ~10 MB + httpx dependency +- **No CLI binary**: No subprocess overhead +- **No Node.js**: No runtime bloat +- **Lightweight**: Ideal for resource-constrained containers + +### Error Handling & Reliability +```python +from anthropic import APIConnectionError, RateLimitError, APITimeoutError + +try: + response = client.messages.create(...) +except RateLimitError: + # Automatic retries (default 2x); configure with max_retries=N + pass +except APITimeoutError: + # Default 10-minute timeout; configure per-request or globally + pass +``` + +--- + +## Comparison Table + +| Feature | Agent SDK | Plain SDK | +|---------|-----------|-----------| +| **Package** | `claude-agent-sdk` 0.2.110 | `anthropic` 0.116.0 | +| **Python** | 3.10+ | 3.9+ | +| **Size** | ~300 MB (bundled binary CLI) | ~10 MB (pure Python) | +| **Node.js required** | No (binary CLI) | No | +| **Agentic loop** | Automatic (query/ClaudeSDKClient) | Manual (messages.create loop) | +| **Tool definition** | `@tool` + in-process MCP | `@beta_tool` decorator | +| **Permission control** | `allowed_tools`, `disallowed_tools`, `permission_mode` | None (list-based inclusion) | +| **Headless autonomous** | Yes (permission_mode='acceptEdits') | Requires manual implementation | +| **Built-in tools** | Read, Write, Edit, Bash, Monitor, Glob, Grep, WebSearch, WebFetch, AskUserQuestion | None (define custom or use tool schema) | +| **Streaming** | Automatic in query loop | Via messages.stream() | +| **Web search/fetch** | Included | Included (webSearch/webFetch tools, since Apr 2026) | +| **Prompt caching** | Not mentioned | Yes, cache_control parameter | +| **Structured outputs** | Not mentioned | Yes, structured_output parameter | +| **Token counting** | Pre-request via query context | messages.count_tokens() + response.usage | +| **Sessions** | Yes, resume=session_id | Manual message history management | +| **Skills loading** | `.claude/skills/*/SKILL.md` convention | N/A | +| **Hooks** | Pre/PostToolUse, SessionStart/End, UserPromptSubmit | N/A | +| **Deployment** | Subprocess (CLI binary via stdio) | In-process HTTP calls | +| **Concurrency** | asyncio-based, one CLI process per query | asyncio-based, HTTP connection pooling | + +--- + +## Recommendation: START WITH AGENT SDK + +### Rationale +1. **Autonomous operation**: `permission_mode='acceptEdits'` + `allowed_tools` = fully headless, no approval prompts. dstack agent runs in a Docker container without user interaction. +2. **Tool loop already solved**: Agent SDK handles the agentic loop internally. You write the objective ("deploy model X"), Agent SDK calls tools, parses responses, repeats until `stop_reason != "tool_use"`. +3. **Built-in tools**: Read, Write, Edit, Bash, WebSearch, WebFetch all pre-integrated; no schema boilerplate. +4. **Permission granularity**: Block dangerous tools, auto-approve safe ones—critical for production agents touching infra. +5. **Skills convention**: `.claude/skills/` can hold domain-specific prompt files (e.g., "dstack-deployment-best-practices.md") loaded automatically. +6. **Observability**: Hooks let you audit every tool call, useful for logging/compliance in a managed service. +7. **No Node.js surprise**: Binary CLI, not JavaScript; deployment burden is bundle size, not runtime complexity. + +### Gotchas / Verification Needed +- **Skill directory loading**: Documentation shows `.claude/skills/*/SKILL.md` convention but does NOT explicitly document programmatic API to point SDK at custom skill directories. May require filesystem setup or `setting_sources` config (not yet verified in official docs). +- **Prompt caching**: Agent SDK docs don't mention cache_control support; unclear if long system prompts benefit from caching. **May need to implement separately via custom MCP server or plain SDK for cost control on repeated deployments.** +- **Concurrent agents**: Each query spawns a CLI subprocess. In a multi-tenant dstack setup, concurrency management (asyncio semaphores) is on you. +- **Model selection**: Agent SDK supports model choice but default strategy unclear. Recommend explicit model selection for cost control (Haiku 4.5 for fast tasks, Sonnet 4.6 for complex deployments). +- **Structured outputs**: Agent SDK doesn't mention structured_output schema support. If the agent must return a validated JSON deployment report, may need to parse text or use plain SDK. + +### Suggested Implementation Pattern (dstack AgentService) + +```python +from claude_agent_sdk import query, ClaudeAgentOptions + +class ClaudeAgentService: + def __init__(self, model: str = "claude-sonnet-4-6"): + self.model = model + + async def deploy(self, objective: str) -> dict: + # Pre-approve safe tools for autonomous operation + options = ClaudeAgentOptions( + allowed_tools=["Read", "Write", "Bash", "WebSearch"], + disallowed_tools=["DeleteFile"], # Custom safety rules + permission_mode="acceptEdits", + system_prompt=f"You are a dstack deployment agent. Objective: {objective}" + ) + + result_text = "" + async for message in query(prompt=objective, options=options): + if hasattr(message, "result"): + result_text = message.result + + return {"success": True, "output": result_text} +``` + +### Defer to Plain SDK If +- Prompt caching becomes critical for cost (long system prompts on every deployment). Plain SDK's `cache_control` is explicit. +- Agent SDK's structured output support lags behind needs (plain SDK has `structured_output` parameter). +- Concurrent session management becomes a bottleneck (plain SDK's HTTP is more amenable to connection pooling). +- Custom tools become complex and in-process MCP not sufficient (plain SDK's tool schema is more flexible). + +### Cost / Token Considerations +- **Agent SDK**: No additional overhead; standard Claude pricing (input/output tokens). CLI binary overhead is one-time disk, not per-request. +- **Plain SDK**: Same; adds prompt caching option (10% cost on cached input). +- **Web search**: Both support. $10 per 1,000 searches. +- **Batch processing**: Not available in Agent SDK (built on CLI); plain SDK offers `messages.batches` for 50% discount on bulk async requests (might matter for large dstack deployments). + +### What to Implement First (Behind AgentService Abstraction) +1. Basic `deploy(objective: str) -> dict` method returning success + logs +2. Tool allowlist (Bash, Read, WebSearch) + blocklist (Delete, Exec) +3. Hook for audit logging of all tool calls +4. Error handling for API key missing (early catch) +5. Model selection flag (default Sonnet 4.6, option for Haiku 4.5 for speed) + +Defer: prompt caching, structured output schema, skill directory custom paths, concurrent session management—validate Agent SDK meets MVP first. + +## Gotchas +**Agent SDK:** +- CLI subprocess per query; no persistent connection pooling. Concurrency scales with asyncio but spawns new processes. In high-throughput scenarios (100+ concurrent deployments), process overhead may become visible. +- Skill directory loading documented as convention (`.claude/skills/*/SKILL.md`) but no explicit programmatic API to restrict sources or load from custom paths. May need filesystem manipulation to use custom skill directories in dstack. +- Prompt caching not mentioned in Agent SDK docs; if long system prompts are repeated per deployment, caching benefit may be missed. Plain SDK's `cache_control` is explicit. +- Structured outputs not mentioned; if deployment report must be guaranteed JSON schema, may need post-processing or custom tool returning JSON string. +- Model selection: Default behavior unclear. Recommend explicit model flag in options to control cost (Haiku 4.5 for fast, Sonnet 4.6 for complex). +- Permission modes work but are SDK-level. If dstack needs finer-grained control (e.g., Bash only in specific directories), hook-based filtering required. + +**Plain SDK:** +- Manual tool-use loop is boilerplate-heavy; easy to introduce bugs (forgetting to append assistant message, wrong tool_result format). Tool runner helper exists but does NOT automate loop—you still implement execution. +- No built-in permission model; must implement allowlist/blocklist at application layer. +- No native session resumption; message history is your responsibility. +- Skills concept doesn't exist; custom tools are pure Python functions or external MCP servers. +- Hooks don't exist; audit logging is manual. +- Lighter footprint but requires more application code to reach Agent SDK feature parity. + +**Both:** +- API key via env var (standard); no option to pass key at init time (security best practice for 12-factor), though both support explicit client = Anthropic(api_key=...). +- No built-in support for multi-replica coordination (e.g., one deployment agent per dstack replica). Session IDs are local; cross-replica session sharing requires external persistence layer. diff --git a/endpoint-implementation-research/config-models.md b/endpoint-implementation-research/config-models.md new file mode 100644 index 0000000000..336b5dcf4b --- /dev/null +++ b/endpoint-implementation-research/config-models.md @@ -0,0 +1,151 @@ +# config-models + +## Summary +Mapped how dstack configuration types are defined and registered end-to-end. All top-level config types live in src/dstack/_internal/core/models/configurations.py, discriminated by a `type: Literal[...]` field, and are aggregated into four unions there (AnyRunConfiguration, AnyApplyConfiguration, BaseApplyConfiguration.__root__, AnyDstackConfiguration) plus an ApplyConfigurationType enum. CLI-side dispatch happens via apply_configurators_mapping in src/dstack/_internal/cli/services/configurators/__init__.py where each configurator class declares TYPE = ApplyConfigurationType.. There is NO generic server-side apply endpoint — each resource type (runs/fleets/gateways/volumes) has its own typed REST router registered in server/app.py and its own APIClient group in dstack/api/server/. No "endpoint" configuration type exists anywhere yet. ProfileParams (including `tags`) is mixed into run configurations only; fleets/gateways/volumes duplicate a subset of fields instead. + +## Key files +- src/dstack/_internal/core/models/configurations.py — RunConfigurationType, BaseRunConfiguration, DevEnvironmentConfiguration, TaskConfiguration, ServiceConfiguration, ServiceConfigurationParams, ReplicaGroup, ProbeConfig, ScalingSpec, RateLimit, AnyRunConfiguration, RunConfiguration, parse_run_configuration, ApplyConfigurationType, AnyApplyConfiguration, BaseApplyConfiguration, parse_apply_configuration, AnyDstackConfiguration, DstackConfiguration — THE registration hub: 4 unions + 1 enum must be touched for a new top-level type (lines 1366-1463). +- src/dstack/_internal/cli/services/configurators/__init__.py — apply_configurators_mapping, run_configurators_mapping, get_apply_configurator_class, load_apply_configuration — CLI dispatch table (lines 27-49) and YAML parse entrypoint (yaml.safe_load -> parse_apply_configuration, lines 62-86). +- src/dstack/_internal/cli/services/configurators/base.py — BaseApplyConfigurator (TYPE: ClassVar[ApplyConfigurationType], apply_configuration, delete_configuration, register_args), ApplyEnvVarsConfiguratorMixin — Abstract base every new configurator subclasses; env-var CLI args + EnvSentinel resolution from os.environ. +- src/dstack/_internal/core/models/profiles.py — ProfileParams (lines 310-493), ProfileParamsConfig, Profile, ProfileRetry, UtilizationPolicy, Schedule, SpotPolicy, CreationPolicy, parse_duration, parse_off_duration, parse_idle_duration — Complete ProfileParams field list incl. tags; mixed into run configs via multiple inheritance. +- src/dstack/_internal/core/models/envs.py — Env, EnvSentinel, EnvVarTuple — Env is a plain pydantic BaseModel with __root__ Union[List[str], Dict[str, Union[str, EnvSentinel]]] — deliberately NOT a CoreModel. +- src/dstack/_internal/core/models/common.py — CoreModel, CoreConfig, generate_dual_core_model, Duration, EntityReference, ApplyAction — pydantic-duality dual models: strict __request__ (extra=forbid) vs __response__ (extra=ignore). Custom Config must go through generate_dual_core_model. +- src/dstack/_internal/core/models/fleets.py — FleetConfiguration, CommonFleetConfigurationProps (type: Literal["fleet"]), BackendFleetConfiguraionProps, SSHFleetConfigurationProps, FleetSpec, Fleet, FleetPlan, ApplyFleetPlanInput, FleetStatus — Non-run config pattern: no ProfileParams inheritance; FleetSpec = configuration + profile + merged_profile. +- src/dstack/_internal/core/models/gateways.py — GatewayConfiguration (type: Literal["gateway"]), GatewaySpec, Gateway, GatewayPlan, ApplyGatewayPlanInput, GatewayStatus — Simplest non-run config: flat CoreModel, no env, no ProfileParams; SUBMITTED/PROVISIONING/RUNNING/FAILED status enum is the closest analog to the planned endpoint lifecycle. +- src/dstack/_internal/core/models/volumes.py — BaseVolumeConfiguration (type: Literal["volume"]), AnyVolumeConfiguration, VolumeConfiguration, parse_volume_configuration, VolumeSpec, Volume, VolumeStatus — Two-stage discriminated parse: `type` then `backend`; explains why BaseApplyConfiguration vs AnyApplyConfiguration differ. +- src/dstack/_internal/cli/services/configurators/gateway.py — GatewayConfigurator (TYPE = ApplyConfigurationType.GATEWAY) — Best template for an async-provisioned resource configurator: get_plan -> confirm -> apply_plan -> poll status until RUNNING/FAILED. +- src/dstack/_internal/cli/services/configurators/run.py — BaseRunConfigurator, TaskConfigurator(:665), DevEnvironmentConfigurator(:680), ServiceConfigurator(:717), interpolate_env(:390) — Run-config CLI flow; ${{ env.* }} interpolation happens CLI-side. +- src/dstack/_internal/cli/commands/apply.py — ApplyCommand — Generic; dispatches on configuration.type via get_apply_configurator_class (line 92). No per-type edits needed. +- src/dstack/_internal/cli/commands/delete.py — DeleteCommand — Same generic dispatch (line 38); requires delete_configuration on the configurator. +- src/dstack/_internal/core/models/services.py — AnyModel, ChatModel, OpenAIChatModel, TGIChatModel, BaseChatModel — The `model` field type of ServiceConfiguration; discriminated on `format`. +- src/dstack/_internal/core/models/runs.py — RunSpec (:522), configuration field (:575), _merged_profile (:590-607) — How AnyRunConfiguration + Profile are merged into merged_profile — the pattern an endpoint spec with ProfileParams should replicate. +- src/dstack/_internal/server/app.py — register_routes (:239-269) — Server-side router registration; a new resource type needs its own router here. +- src/dstack/api/server/__init__.py — APIClient properties: fleets, runs, gateways, volumes, ... (:76-146) — Low-level HTTP client groups (dstack/api/server/_*.py); a new resource needs a new group. +- scripts/docs/gen_schema_reference.py — generate_schema_reference, sub_schema_reference — mkdocs hook expanding '#SCHEMA# dotted.model.Path' directives in mkdocs/docs/reference/dstack.yml/*.md. +- .github/workflows/build-artifacts.yml — line 248: DstackConfiguration.schema_json() — CI generates configuration.json editor schema from DstackConfiguration — picks up a new type automatically once added to AnyDstackConfiguration. + +## Details +# Configuration types in dstack — ground truth (verified 2026-07-03, master @ 28ea5f86f) + +No `endpoint` configuration type, `EndpointConfiguration` class, or `ENDPOINT` enum member exists anywhere in the codebase. `grep` confirms. + +## 1. Run configurations — `src/dstack/_internal/core/models/configurations.py` (1463 lines) + +### Discriminator & enums +- `RunConfigurationType(str, Enum)` (:77-80): `DEV_ENVIRONMENT = "dev-environment"`, `TASK = "task"`, `SERVICE = "service"`. +- Every configuration class carries `type: Literal[""] = ""` used as the pydantic discriminator. + +### Base class +`class BaseRunConfiguration(CoreModel)` (:484) — `type: Literal["none"]` (overridden by subclasses). Fields: +- `name: Optional[str] = None`, `image: Optional[str] = None`, `user: Optional[str] = None`, `privileged: bool = False`, `entrypoint: Optional[str] = None`, `working_dir: Optional[str] = None`, `home_dir: str = "/root"` (deprecated), `registry_auth: Optional[RegistryAuth] = None`, `python: Optional[PythonVersion] = None`, `nvcc: Optional[bool] = None`, `single_branch: Optional[bool] = None`, `env: Env = Env()` (:540-543), `shell: Optional[str] = None`, `resources: ResourcesSpec = ResourcesSpec()` (:554-556), `priority: Optional[int]` (0..100), `volumes: List[MountPoint] = []`, `docker: Optional[bool] = None`, `repos: list[RepoSpec] = []`, `files: list[FilePathMapping] = []`, `setup: CommandsList = []` (deprecated). +- Validators coerce strings: `volumes` via `parse_mount_point`, `files` via `FilePathMapping.parse`, `repos` via `RepoSpec.parse`; mutual exclusions for image/python/docker/nvcc. + +### Mixin param classes +- `ConfigurationWithPortsParams` (:657): `ports: List[Union[ValidPort, constr(regex=...), PortMapping]] = []` (`ValidPort = conint(gt=0, le=65536)` :53). +- `ConfigurationWithCommandsParams` (:672): `commands: CommandsList = []` + root_validator requiring `commands` or `image` (skipped when `replicas` is a list). +- `DevEnvironmentConfigurationParams` (:687): `ide` (vscode/cursor/windsurf/zed), `version`, `init: CommandsList`, `inactivity_duration`. +- `TaskConfigurationParams` (:768): `nodes: int = 1 (ge=1)`. + +### Concrete classes (note MRO — ProfileParams first) +- `DevEnvironmentConfiguration(ProfileParams, BaseRunConfiguration, ConfigurationWithPortsParams, DevEnvironmentConfigurationParams, generate_dual_core_model(DevEnvironmentConfigurationConfig))` (:752) — `type: Literal["dev-environment"] = "dev-environment"` (:759). +- `TaskConfiguration(ProfileParams, BaseRunConfiguration, ConfigurationWithCommandsParams, ConfigurationWithPortsParams, TaskConfigurationParams, generate_dual_core_model(TaskConfigurationConfig))` (:782) — `type: Literal["task"] = "task"` (:790). +- `ServiceConfiguration(ProfileParams, BaseRunConfiguration, ConfigurationWithCommandsParams, ServiceConfigurationParams, generate_dual_core_model(ServiceConfigurationConfig))` (:1328) — `type: Literal["service"] = "service"` (:1335); property `replica_groups -> List[ReplicaGroup]` (:1337-1363). + +Each has a paired `*Config` class (e.g. `ServiceConfigurationConfig` :1316) that composes `schema_extra` from `ProfileParamsConfig`, `BaseRunConfigurationConfig`, `ServiceConfigurationParamsConfig` — needed because pydantic-duality requires Config passed via `generate_dual_core_model()`. + +### ServiceConfiguration own fields (`ServiceConfigurationParams` :961-1313) +- `port: Union[ValidPort, constr(regex=r"^[0-9]+:[0-9]+$"), PortMapping]` — REQUIRED; validator coerces int → `PortMapping(local_port=80, container_port=v)` (:1050-1056). +- `gateway: Optional[Union[bool, EntityReference, str]] = None` (str coerced to EntityReference). +- `strip_prefix: bool = True`. +- `model: Optional[AnyModel] = None` (:993-1003); validator `convert_model` (:1058-1062): str → `OpenAIChatModel(type="chat", name=v, format="openai")`. `AnyModel = Union[ChatModel]`; `ChatModel = Annotated[Union[TGIChatModel, OpenAIChatModel], Field(discriminator="format")]` (core/models/services.py:75-76). `OpenAIChatModel` has `prefix: str = "/v1"`. +- `https: Optional[Union[bool, Literal["auto"]]] = None`. +- `auth: bool = True`. +- `scaling: Optional[ScalingSpec] = None` (`ScalingSpec` :213 — metric Literal["rps"], target float, window, scale_up_delay/scale_down_delay Durations). +- `rate_limits: list[RateLimit] = []` (:282). +- `probes: Optional[list[ProbeConfig]] = None` (:1019-1026) — `None` = may get default when `model` set; `[]` = explicitly none. `ProbeConfig` (:365): `type: Literal["http"]`, `url` (default `/`), `method` (get/post/put/delete/patch/head), `headers: list[HTTPHeaderSpec]`, `body`, `timeout` (default 10s), `interval` (default 15s), `ready_after` (default 1), `until_ready` (default false). The default model probe is built server-side in `server/services/jobs/configurators/base.py::_openai_model_probe_spec` (:472-491): POST `{prefix}/chat/completions` with `{"model": name, "messages":[{"role":"user","content":"hi"}], "max_tokens":1}`, timeout `OPENAI_MODEL_PROBE_TIMEOUT = 30`. +- `replicas: Optional[Union[List[ReplicaGroup], Range[int]]] = None` (:1028-1040); `ReplicaGroup` (:817) has name, count: Range[int], scaling, resources, spot_policy, reservation, commands, image, python, nvcc, docker, privileged, router. Multiple root_validators enforce group vs top-level mutual exclusions (:1123-1313). +- `router: Optional[AnyServiceRouterConfig] = None` (from `core/models/routers.py`). +- `resources` (inherited): `ResourcesSpec` (core/models/resources.py:377) — cpu, memory, shm_size, gpu, disk. + +### Unions & parsers (the registration hub) +- `AnyRunConfiguration = Union[DevEnvironmentConfiguration, TaskConfiguration, ServiceConfiguration]` (:1366). +- `class RunConfiguration(CoreModel)` (:1369): `__root__: Annotated[AnyRunConfiguration, Field(discriminator="type")]`. +- `def parse_run_configuration(data: dict) -> AnyRunConfiguration` (:1376-1381) — raises `ConfigurationError` on ValidationError. +- `class ApplyConfigurationType(str, Enum)` (:1384-1390): DEV_ENVIRONMENT, TASK, SERVICE, FLEET, GATEWAY, VOLUME. +- `AnyApplyConfiguration = Union[AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, AnyVolumeConfiguration]` (:1393-1398). +- `class BaseApplyConfiguration(CoreModel)` (:1401-1421): `__root__: Annotated[Union[AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, BaseVolumeConfiguration], Field(discriminator="type")]` — note `BaseVolumeConfiguration` here (not the backend union) because volumes need a second parse on `backend`. +- `def parse_apply_configuration(data: dict) -> AnyApplyConfiguration` (:1424-1437): pass 1 with `BaseApplyConfiguration.__response__` (extra ignored) to find the type; if not a volume, pass 2 strict (`BaseApplyConfiguration.parse_obj`, extra forbidden) for validation, return pass-1 object; volumes delegate to `parse_volume_configuration` (volumes.py:191). +- `AnyDstackConfiguration = Union[AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, VolumeConfiguration]` (:1440-1445) — uses the `VolumeConfiguration` ROOT model, not `AnyVolumeConfiguration`. +- `class DstackConfiguration(CoreModel)` (:1448-1463): root over `AnyDstackConfiguration` with draft-07 `$schema` + `additionalProperties: True` — used ONLY by CI (`.github/workflows/build-artifacts.yml:248`: `python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json`). + +## 2. Non-run configuration models + +### Fleets — `core/models/fleets.py` +- `CommonFleetConfigurationProps` (:211): `type: Literal["fleet"] = "fleet"`, `name`, `placement`, `blocks`. +- `BackendFleetConfiguraionProps` (:232 — NOTE the typo "Configuraion" is the real class name): nodes (`FleetNodesSpec`), reservation, resources, backends, regions, availability_zones, instance_types, spot_policy, retry, max_price, idle_duration, `tags: Optional[Dict[str,str]]` (:298), backend_options. These duplicate ProfileParams-ish fields — fleets do NOT inherit ProfileParams. +- `SSHFleetConfigurationProps` (:345): ssh_config, `env: Env = Env()`. +- `FleetConfiguration(SSHFleetConfigurationProps, BackendFleetConfiguraionProps, CommonFleetConfigurationProps, generate_dual_core_model(FleetConfigurationConfig))` (:362). +- `FleetSpec` (:393): `configuration: FleetConfiguration`, `configuration_path: Optional[str]`, `profile: Profile`, `merged_profile: Annotated[Profile, Field(exclude=True)]` computed by root_validator `_merged_profile` (:408-424) — loops `for key in ProfileParams.__fields__` and overrides profile values with non-None config values. +- Runtime model `Fleet` (:427): id, name, project_name, spec, created_at, `status: FleetStatus` (SUBMITTED/ACTIVE/TERMINATING/TERMINATED/FAILED :34-41 — comment says SUBMITTED/FAILED reserved for async processing), status_message, instances. Plan models: `FleetPlan` (:438), `ApplyFleetPlanInput` (:456). + +### Gateways — `core/models/gateways.py` +- `GatewayConfiguration(CoreModel)` (:59): `type: Literal["gateway"] = "gateway"`, name, default, `backend: BackendType` (required), `region: str` (required), instance_type, router, domain, public_ip, certificate, replicas, `tags` (:111). Flat class — no ProfileParams, no env, no resources. +- `GatewaySpec` (:125): configuration + configuration_path only. +- `GatewayStatus` (:17): SUBMITTED/PROVISIONING/RUNNING/FAILED — closest analog for an async-provisioned endpoint lifecycle. Runtime `Gateway` (:141) has status, status_message, created_at. +- `GatewayPlan` (:176), `ApplyGatewayPlanInput` (:185). + +### Volumes — `core/models/volumes.py` +- `BaseVolumeConfiguration(CoreModel)` (:36): `type: Literal["volume"] = "volume"`, `backend: Any` (overridden per subclass with `Literal[BackendType.X]`), name, size, auto_cleanup_duration, `tags` (:59). +- Backend subclasses: `AWSVolumeConfiguration` (:113), `GCPVolumeConfiguration` (:121), `RunpodVolumeConfiguration` (:129), `KubernetesVolumeConfiguration` (:137); `AnyVolumeConfiguration` union (:179); `VolumeConfiguration` root model discriminated on `backend` (:187); `parse_volume_configuration` (:191). +- `VolumeStatus` (:19): SUBMITTED/PROVISIONING/ACTIVE/FAILED with `finished_statuses()`. +- `VolumeSpec` (:198): configuration + configuration_path. + +### Key difference from run configs +Non-run configs have `name` for the resource itself, plain status enums, `Spec` wrapper `{configuration, configuration_path[, profile]}`, `Plan`, and `ApplyPlanInput{spec, current_resource}` models. Server exposes per-type REST endpoints; CLI configurators call `self.api.client..get_plan/apply_plan/delete` and poll. + +## 3. ProfileParams — `core/models/profiles.py:310-493` (complete) +`backends: Optional[List[BackendType]]`, `regions: Optional[List[str]]`, `availability_zones: Optional[List[str]]`, `instance_types: Optional[List[str]]`, `reservation: Optional[str]`, `spot_policy: Optional[SpotPolicy]` (spot/on-demand/auto), `retry: Optional[Union[ProfileRetry, bool]]`, `max_duration: Optional[Union[Literal["off"], int]]`, `stop_duration: Optional[Union[Literal["off"], int]]`, `max_price: Optional[float] (gt=0)`, `creation_policy: Optional[CreationPolicy]` (reuse/reuse-or-create), `idle_duration: Optional[int]`, `utilization_policy: Optional[UtilizationPolicy]`, `startup_order: Optional[StartupOrder]`, `stop_criteria: Optional[StopCriteria]`, `schedule: Optional[Schedule]` (cron), `fleets: Optional[list[Union[EntityReference, str]]]`, `instances: Optional[List[InstanceSelector]] (min_items=1)`, `tags: Optional[Dict[str, str]]` (:462-471, validated by `tags_validator`), `backend_options: Optional[List[AnyBackendProfileOptions]]`. All Optional/None-default. Duration validators pre-parse "2h"/"off" strings. `Profile = ProfileProps(name, default) + ProfileParams` (:514). `ProfileParamsConfig.schema_extra` (:289) adds string/bool alt types for max_duration/stop_duration/idle_duration/instances. + +## 4. Env — `core/models/envs.py` +`class Env(BaseModel)` (:42) — custom root model: `__root__: Union[List[str matching ^([a-zA-Z_][a-zA-Z0-9_]*)(=.*$|$)], Dict[str, Union[str, EnvSentinel]]] = {}`. Docstring explicitly: NOT a CoreModel because pydantic-duality doesn't play well with custom root models. List form normalized to dict by validator; bare `VAR` becomes `EnvSentinel(key=VAR)`. API: `as_dict()` (raises ValueError if sentinels unresolved), `update()`, `keys/values/items`, `__getitem__/__setitem__`, dict-like `copy()`. Sentinel resolution from `os.environ` happens CLI-side in `ApplyEnvVarsConfiguratorMixin.apply_env_vars` (cli/services/configurators/base.py:95-103); `-e KEY[=VALUE]` args registered by `register_env_args` (:83-93). + +## 5. Tags +Yes, supported: `ProfileParams.tags` gives all run configurations tags for free; fleets (fleets.py:298), gateways (gateways.py:111), volumes (volumes.py:59) declare their own. Validation: `src/dstack/_internal/utils/tags.py::tags_validator` — key `^[_\-a-zA-Z0-9]{1,60}$`, value `^[a-zA-Z0-9 .:/=_\-+@]{0,256}$`. No other "metadata" field exists on configurations. + +## 6. YAML parsing/validation flow for `dstack apply` +1. `ApplyCommand._command` (cli/commands/apply.py:72) → `load_apply_configuration(args.configuration_file)` (cli/services/configurators/__init__.py:62-86): resolves `$PWD/.dstack.yml`/`.yaml` or `-f FILE` or stdin (`-`), `yaml.safe_load`, then `parse_apply_configuration(dict)`. +2. `get_apply_configurator_class(configuration.type)` (:52-55) — `ApplyConfigurationType(configurator_type)` then dict lookup in `apply_configurators_mapping` (:27-39, built from class list `[DevEnvironmentConfigurator, TaskConfigurator, ServiceConfigurator, FleetConfigurator, GatewayConfigurator, VolumeConfigurator]` keyed by `cls.TYPE`). +3. Configurator instantiated with `api_client=self.api` (`dstack.api._public.Client`), `configurator.get_parser()` parses per-type extra args, then `apply_configuration(conf, configuration_path, command_args, configurator_args)`. +4. `dstack delete/destroy` (cli/commands/delete.py:35-44) uses the same dispatch and calls `delete_configuration`. +5. `dstack apply -h TYPE` (apply.py:27-36) parses TYPE via the `ApplyConfigurationType` enum. +6. Run-config interpolation: `BaseRunConfigurator.interpolate_env` (run.py:390-408) uses `VariablesInterpolator({"env": ...}, skip=["secrets"])` on registry_auth and probe fields. + +Server side: there is NO generic apply/config endpoint. Runs: client sends `RunSpec` (core/models/runs.py:522; `configuration: Annotated[AnyRunConfiguration, Field(discriminator="type")]` :575; `merged_profile` root_validator :590-607) to `runs` router. Fleets/gateways/volumes have their own typed routers. Routers registered in `server/app.py::register_routes` (:239-269): server, users, auth, projects, backends, fleets, instances, repos, runs, gpus, metrics, logs, secrets, gateways, volumes, service_proxy, model_proxy, prometheus, files, events, templates, exports, imports, sshproxy, public_keys. Low-level client groups: `dstack/api/server/_*.py` with properties on `APIClient` (api/server/__init__.py:76-146). Run-type-only server dispatch: `server/services/jobs/__init__.py::_get_job_configurator` (:316-333) maps `RunConfigurationType` → `{DevEnvironmentJobConfigurator, TaskJobConfigurator, ServiceJobConfigurator}` (server/services/jobs/configurators/{dev,task,service}.py each with `TYPE: RunConfigurationType`). + +## 7. EXHAUSTIVE checklist to add a new top-level config type `endpoint` + +Core models: +1. `src/dstack/_internal/core/models/configurations.py` — define `EndpointConfiguration` (with `type: Literal["endpoint"] = "endpoint"`); add `ENDPOINT = "endpoint"` to `ApplyConfigurationType` (:1384); add the class to `AnyApplyConfiguration` (:1393), to `BaseApplyConfiguration.__root__` union (:1411), and to `AnyDstackConfiguration` (:1440). (Or define the model in a new `core/models/endpoints.py` and import it, like fleets/gateways/volumes do — beware circular imports: configurations.py already imports fleets/gateways/volumes/profiles.) +2. If it's a standalone resource (like gateway/volume): also define `EndpointSpec {configuration, configuration_path}`, `Endpoint` (runtime: id, name, project_name, created_at, status, status_message, ...), `EndpointStatus` enum (mirror `GatewayStatus`: SUBMITTED/PROVISIONING/RUNNING(ACTIVE)/FAILED), `EndpointPlan`, `ApplyEndpointPlanInput`. + +CLI: +3. New `src/dstack/_internal/cli/services/configurators/endpoint.py`: `class EndpointConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[EndpointConfiguration])` with `TYPE = ApplyConfigurationType.ENDPOINT`, implementing `apply_configuration()` and `delete_configuration()` (both abstract), optionally `register_args()`/`apply_args()`. GatewayConfigurator (gateway.py:37) is the closest template (async provisioning + status polling). +4. `src/dstack/_internal/cli/services/configurators/__init__.py` — add `EndpointConfigurator` to the class list building `apply_configurators_mapping` (:30-39). Do NOT add to `run_configurators_mapping` (run types only). `dstack apply`/`dstack delete`/`dstack apply -h endpoint` then work with no further CLI edits. + +Server (endpoint-as-own-resource path): +5. New router `src/dstack/_internal/server/routers/endpoints.py` + registration in `server/app.py::register_routes` (:239). +6. New `src/dstack/api/server/_endpoints.py` APIClient group + property in `src/dstack/api/server/__init__.py` (pattern :96-131), so the CLI configurator can call `self.api.client.endpoints.*`. +7. (Server services/DB/background processing are separate topics — not covered here.) + +Docs/schema: +8. `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._internal.core.models....EndpointConfiguration` directives (processed by `scripts/docs/gen_schema_reference.py`). +9. `mkdocs.yml` nav (:343-349) — add `- endpoint: docs/reference/dstack.yml/endpoint.md`. +10. `mkdocs/docs/reference/dstack.yml.md` index — add link to the new page. +11. Editor JSON schema (`.github/workflows/build-artifacts.yml:248`) regenerates automatically from `DstackConfiguration` once step 1 includes the type in `AnyDstackConfiguration`. + +Optional: +12. `src/dstack/api/__init__.py` — public alias (pattern: `Service = _ServiceConfiguration` :28-30). +13. If ProfileParams is inherited, also create `EndpointConfigurationConfig(ProfileParamsConfig)` composing `schema_extra` (pattern: `ServiceConfigurationConfig` :1316-1325) and pass it via `generate_dual_core_model(...)` as the last base class. + +## Gotchas +1) Pydantic v1 syntax throughout (`root_validator`/`validator`, `__root__` models, `constr(regex=...)`) — do not write v2-style code. 2) pydantic-duality: every CoreModel is dual (`__request__` extra=forbid / `__response__` extra=ignore). NEVER define `class Config` directly on a model — it breaks `__response__`; pass a custom Config class via `generate_dual_core_model(MyConfig)` as a BASE CLASS (last in MRO), and when combining mixins, the Config class must manually chain each parent's `schema_extra` (see ServiceConfigurationConfig at configurations.py:1316). 3) `BaseRunConfiguration.type` is `Literal["none"]` with NO default — every concrete subclass overrides it with its own Literal + default; the discriminator for apply configs is `type`, and volumes have a SECOND discriminator `backend` requiring the two-stage `parse_apply_configuration` (first pass parses with `__response__` i.e. extras ignored, second strict pass validates). A new type with only `type` as discriminator goes in the "Final configurations" part of `BaseApplyConfiguration.__root__`. 4) `AnyDstackConfiguration` (editor JSON schema union) uses the wrapped `VolumeConfiguration` root model, not `AnyVolumeConfiguration` — subtle asymmetry with `AnyApplyConfiguration`. 5) MRO matters: run configs list `ProfileParams` FIRST (`ServiceConfiguration(ProfileParams, BaseRunConfiguration, ...)`); ProfileParams already contains `tags`, `schedule`, `fleets`, `idle_duration` etc., so an endpoint config inheriting ProfileParams gets all of those including tags — no need to re-declare. 6) Fleet/gateway/volume configs do NOT inherit ProfileParams (fleets duplicate ~12 of its fields in `BackendFleetConfiguraionProps` — note the real class name has the typo "Configuraion"); only run configs use the merged_profile pattern (`RunSpec._merged_profile` runs.py:590 iterates `ProfileParams.__fields__`). 7) `Env` is NOT a CoreModel (plain BaseModel with custom root) and `EnvSentinel` values (bare `VAR` entries) are resolved from os.environ CLI-side only (`ApplyEnvVarsConfiguratorMixin.apply_env_vars`); a server-side agent creating configs from an endpoint's env must handle/forbid unresolved sentinels itself (`Env.as_dict()` raises ValueError on unresolved). 8) `get_apply_configurator_class` does `ApplyConfigurationType(configurator_type)` — forgetting the enum member makes `dstack apply` crash with ValueError before any useful message. 9) `ServiceConfiguration.port` is REQUIRED (no default) and coerced to `PortMapping(local_port=80, ...)`; `model: str` is coerced to `OpenAIChatModel(type="chat", format="openai", prefix="/v1")` — a preset/agent-generated service config must set `port` and should set `model` for the OpenAI-compatible proxy + default `/v1/chat/completions` probe (probe default is applied server-side in `server/services/jobs/configurators/base.py::_openai_model_probe_spec`, not in the model). 10) Old-style server background tasks were reworked: `server/background/pipeline_tasks` + `server/background/scheduled_tasks` (see app.py:25-27 imports `start_pipeline_tasks`, `start_scheduled_tasks`) — do not reference a `background/tasks` module. 11) The CLI never sends `AnyApplyConfiguration` to the server; each type maps to its own typed REST API (RunSpec/FleetSpec/GatewaySpec/VolumeSpec + plan/apply-plan endpoints), so a new endpoint type needs its own router + API client group, not a hook into an existing generic endpoint. 12) `dstack delete` requires `delete_configuration` implemented (abstract), and gateway/volume configurators poll async deletion — deletion of the new resource should follow the same async pattern. diff --git a/endpoint-implementation-research/critique.md b/endpoint-implementation-research/critique.md new file mode 100644 index 0000000000..f322f71353 --- /dev/null +++ b/endpoint-implementation-research/critique.md @@ -0,0 +1,63 @@ +# Completeness critique + +## Gaps +### How is an endpoint named and how does its name map to a run name? Verified: ProfileParams (src/dstack/_internal/core/models/profiles.py:280-470) has NO `name` field, so the proposed config {type, model, env, ProfileParams} has no identity; run_name regex is ^[a-z][a-z0-9-]{1,40}$ while model ids are like `Qwen/Qwen3-32B` (uppercase, slash). Need: explicit `name` field on EndpointConfiguration (copy BaseRunConfiguration.name or FleetConfiguration.name pattern), a deterministic endpoint->run_name derivation/sanitization, and per-project uniqueness semantics (unique constraint vs volumes-style advisory-lock dance). +- Why: Without a naming scheme the plan cannot define the DB unique constraint, the endpoint->run linkage, or CLI apply/delete addressing; and submit_run silently deletes finished runs with a colliding run_name. +- Where: src/dstack/_internal/core/models/configurations.py (BaseRunConfiguration.name validator), src/dstack/_internal/core/models/fleets.py, src/dstack/_internal/server/services/volumes.py name-uniqueness dance + +### What is the concrete API for preset-vs-fleet matching, i.e. 'does an existing fleet have capacity for this preset'? Candidates verified to exist but not researched: services/runs/plan.py get_plan path (fleet-aware: combine_fleet_and_run_profiles, combine_fleet_and_run_requirements, check_can_create_new_cloud_instance_in_fleet at plan.py:37-41,64-65), get_offers_by_requirements(project, profile, requirements, ...) (services/offers.py:30), get_pool_instances (services/instances.py:723). Unknown: whether RunPlan/JobPlan offers (core/models/runs.py:708-722) distinguish 'reuses idle fleet instance' from 'provisions new cloud instance' (InstanceOfferWithAvailability.availability semantics), and whether calling get_plan from a background task is safe (session/commit behavior). +- Why: Preset selection 'matching existing fleets' is a core requirement; without the exact offer/availability semantics the plan will hallucinate a matching API or pick presets that trigger unwanted cloud provisioning. +- Where: src/dstack/_internal/server/services/runs/plan.py, src/dstack/_internal/server/services/offers.py, src/dstack/_internal/core/models/instances.py (InstanceAvailability enum), src/dstack/_internal/server/services/instances.py:723 + +### Preset storage format and source of truth are undecided: full ServiceConfiguration YAML vs partial dict, matching keys (model id normalization, GPU requirement expression), and where presets live — vendored files under src/dstack (ships in wheel), DB rows, or a git repo like the existing UITemplate system (core/models/templates.py, services/templates.py, DSTACK_SERVER_TEMPLATES_REPO). Note replicas can be a list of replica-group objects, and `model:` string coerces to OpenAIChatModel — preset schema must round-trip these. +- Why: The plan must specify preset schema, lookup order (preset vs agent fallback), and packaging; the existing templates feature also creates a naming/namespace collision to resolve deliberately. +- Where: src/dstack/_internal/core/models/templates.py, src/dstack/_internal/server/services/templates.py, src/dstack/_internal/server/routers/templates.py + +### The claude-agent-sdk Python API surface is UNVERIFIED prose: exact symbols (query() vs ClaudeSDKClient, ClaudeAgentOptions fields like allowed_tools/permission_mode/max_turns/cwd/model, custom in-process tool definition / SDK MCP server API, structured result extraction) were not pinned against the real package, nor was its footprint in the dstack server images (bundled CLI binary ~300MB, subprocess spawning inside the server container). +- Why: The agent-launch step is the least-verified part of the plan; hallucinated SDK calls or an image that can't host the bundled CLI would invalidate the whole agent path. A fallback decision (plain anthropic SDK loop) hinges on these facts. +- Where: PyPI claude-agent-sdk (pip download + inspect), https://code.claude.com/docs/en/agent-sdk/, docker/server/release/ and docker/server/Dockerfile.nebius, pyproject.toml optional-dependencies pattern (S3Storage try/except ImportError precedent) + +### Where does the Anthropic API key come from — a DSTACK_SERVER_* module constant in server/settings.py (documented in mkdocs/docs/reference/env.md) or a per-project encrypted secret via the verified secrets service (src/dstack/_internal/server/services/secrets.py:29-242, get_project_secrets_mapping/create_or_update_secret, EncryptedString)? Also: behavior when the key is absent (endpoint FAILED at submit vs feature-flag-gated router). +- Why: Determines multi-tenant cost attribution, encryption-at-rest of the key, and the failure mode for servers without a key; the plan must pick one and specify the setting name/doc update. +- Where: src/dstack/_internal/server/services/secrets.py, src/dstack/_internal/server/settings.py, src/dstack/_internal/settings.py FeatureFlags + +### What tool surface does the agent get to author/deploy the service? Options with different verified constraints: (a) in-process Python tools wrapping services.runs.apply_plan/submit_run (commits sessions multiple times — cannot run inside the pipeline worker's fetch session), (b) HTTP calls to the server's own REST API (needs a user token — UserModel token is encrypted; how to mint/decrypt one for the agent), (c) shelling out to the dstack CLI (is the CLI + config file even available/configured in the server container?). None of this was researched. +- Why: This is the central integration decision; each option changes auth, transaction handling, and idempotency, and the plan cannot be followed without picking one and naming exact functions. +- Where: src/dstack/_internal/server/services/runs/__init__.py (apply_plan/submit_run session semantics), src/dstack/_internal/server/models.py UserModel.token/EncryptedString, src/dstack/_internal/server/services/users.py (token access), docker/server/ + +### How is agent progress persisted for crash recovery? Pipeline crash-recovery re-runs the whole process() step after lock expiry, and Agent SDK sessions are replica-local (no cross-replica resume). Undecided: does EndpointModel persist an attempt counter + agent output (Text column with pydantic JSON, the established pattern), and does the agent step begin with reconciliation (get_run_by_name for the endpoint's derived run name; check RunModel.deleted at models.py:411) before launching a new agent session? +- Why: Without a persisted state machine + reconcile-first design, a replica death mid-agent-run duplicates deployments and Anthropic spend; the plan needs the exact EndpointModel columns and status enum values. +- Where: src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py (chunked state machine precedent), src/dstack/_internal/server/services/runs/__init__.py get_run_by_name + +### Out-of-band run lifecycle reconciliation: what happens when a user stops/deletes the endpoint's underlying run directly (stop_runs/delete_runs are public APIs; RunModel soft-deletes via `deleted`, models.py:411), or when the run goes FAILED/TERMINATED after the endpoint is ACTIVE? Does the endpoint pipeline keep re-checking active endpoints (min_processing_interval) and transition to FAILED / re-provision, and does endpoint deletion call stop_runs+delete_runs or leave the run? +- Why: Defines the steady-state half of the state machine and the delete flow; skipping it leaves dangling endpoints pointing at deleted runs. +- Where: src/dstack/_internal/server/services/runs/__init__.py stop_runs/delete_runs, src/dstack/_internal/server/background/pipeline_tasks/gateways.py (parent-polls-child precedent) + +### Concrete ACTIVE/FAILED criteria and deadlines: mechanism is established (JobModel.registered gated by probes; probes never fail a job), but the plan needs exact queries — how the endpoint pipeline finds the run's latest jobs and reads `registered` (RunModel.jobs relationship / latest submission filter), what provisioning deadline default applies (and its interaction with RunModel.resubmission_attempt retries), and whether 'active' means >=1 registered replica or all replicas. +- Why: Health-checking is a stated requirement; without the exact registered-job query and a timeout value the developer must re-research, and endpoints could stay 'provisioning' forever. +- Where: src/dstack/_internal/server/models.py RunModel.jobs/JobModel, src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py:1119-1130 is_job_ready, src/dstack/_internal/server/background/pipeline_tasks/runs/active.py + +### What endpoint URL is surfaced to the user as the OpenAI-compatible base URL, and how does the server know its own external base URL? ServiceSpec/model URL construction in _register_service_in_server (services/services/__init__.py:275) vs gateway domain for gateway-backed services — the exact setting (e.g. a SERVER_URL-like constant) and the returned Endpoint model fields (base_url, model name for /v1/models) were not established. +- Why: The whole point of an endpoint is a usable URL; the plan must specify what `dstack apply` prints and what the GET /endpoints API returns for both gateway and in-server-proxy cases. +- Where: src/dstack/_internal/server/services/services/__init__.py, src/dstack/_internal/server/settings.py, src/dstack/_internal/core/models/runs.py ServiceSpec/ServiceModelSpec + +### CLI/API surface decisions not pinned: does the endpoint get a server-side get_plan/apply_plan pair (gateway pattern, recommended by blueprint) or client-side plan (volume pattern); which permission dep (ProjectMember vs ProjectAdmin — precedents differ); is there a `dstack ps`-style listing or only apply/delete; and is the endpoints spec added to the plugins ApplySpec TypeVar (src/dstack/plugins/_models.py:8) or deliberately excluded? +- Why: These are one-line decisions but each one, if unspecified, forces the implementer to re-derive the pattern; the plugins TypeVar in particular breaks type-checking if forgotten. +- Where: src/dstack/_internal/server/routers/gateways.py, src/dstack/_internal/server/security/permissions.py, src/dstack/plugins/_models.py + +### What instructional context does the embedded agent receive and from where at runtime? Established: nothing agent-usable ships in the wheel (skills/, mkdocs/ excluded) and recipes need two consumption strategies (recipes.vllm.ai/models.json JSON vs SGLang MDX). Undecided: the exact vendored path under src/dstack/_internal/... for prompt/skill content, whether the agent may WebFetch at runtime (server egress policy, pytest --disable-socket means all of it must be mockable), and content update cadence/licensing attribution placement. +- Why: Agent quality depends entirely on this context; a plan that says 'load the skill' without a shippable path and a network-or-vendored decision is unimplementable. +- Where: src/dstack/_internal/server/ (pick vendoring location), pyproject.toml hatchling packaging config, skills/dstack/SKILL.md (source material to distill) + +## Risks +- Long-running agent vs replica death: heartbeater allows unbounded process(), but a crash re-runs the entire step from scratch on another replica with no agent-session resume; the endpoint state machine must be reconcile-before-act (look up existing run by derived name, check RunModel.deleted/status) and cap attempts, or crashes duplicate deployments. +- Unbounded Anthropic spend: pipeline retry semantics (row re-fetched after lock expiry, min_processing_interval re-processing) plus a flaky agent = a paid crash loop; the plan must specify max_turns/model choice, a persisted attempt counter, and a terminal FAILED state that stops retries. +- Transaction hazard: submit_run/apply_plan commit the session multiple times and take advisory locks; calling them inside the pipeline worker's token-guarded update flow breaks the lock protocol — the worker must use a fresh get_session_ctx() for run submission and treat 'run created but endpoint row update lost lock' as a reconcilable state, not corruption. +- Security boundary: the Agent SDK spawns a CLI subprocess with tool execution inside the multi-tenant dstack server container (which holds DB creds, encryption keys, cloud creds in env); tool allowlisting/permission mode and env scrubbing for the subprocess must be designed, or prompt-injected recipe content can exfiltrate server secrets. +- Event-loop starvation: agent subprocess waits and Anthropic HTTP calls run for minutes; routing them through the shared 128-thread default executor (app.py:123) or blocking the loop stalls heartbeats for ALL pipelines on the replica — endpoints pipeline needs native-async subprocess handling or a dedicated executor, and a small workers_num to bound concurrent agent runs. +- Silent history loss: submit_run deletes any finished run with the same run_name — a deterministic endpoint-derived run name erases previous attempts' logs/history on every retry; decide between attempt-suffixed names and accepting the erasure. +- Health-check false positives/negatives: RunStatus.RUNNING fires on any RUNNING replica (not model-ready), probes never fail a job (unhealthy service stays RUNNING with registered=False forever), and TGI-format models get no default probe — the endpoint needs its own registered-based check AND its own deadline to reach FAILED. +- register_service raises synchronously inside submit_run (gateway missing, FORBID_SERVICES_WITHOUT_GATEWAY, autoscaling without gateway) — agent- or preset-generated ServiceConfigurations must handle this as a submission failure path, and the agent context must know whether the target project has a gateway. +- Preset/fleet drift: a preset matched against a fleet at submit time can be invalidated before provisioning completes (fleet deleted, instances busy); matching must be advisory (offers-based) with the run's own scheduling as the source of truth, not a hard endpoint->fleet binding. +- Namespace collision: an existing 'templates' feature (UITemplate, /templates/list, DSTACK_SERVER_TEMPLATES_REPO) already occupies the config-template concept; presets must either reuse it or pick distinct naming to avoid API/docs confusion. +- Testing constraint shapes the design: pytest runs with --disable-socket and schema comes from metadata.create_all (missing migration won't fail tests), so the agent client must be injectable/mockable and the alembic migration (expand-and-contract, one table, partial pipeline-fetch index with both postgresql_where and sqlite_where) must be an explicit plan step with its own review. diff --git a/endpoint-implementation-research/examples-llm-deployments.md b/endpoint-implementation-research/examples-llm-deployments.md new file mode 100644 index 0000000000..46f4fceb7e --- /dev/null +++ b/endpoint-implementation-research/examples-llm-deployments.md @@ -0,0 +1,129 @@ +# examples-llm-deployments + +## Summary +The repo's LLM serving examples now live primarily as YAML embedded in mkdocs docs pages (mkdocs/docs/examples/inference/{vllm,sglang,trtllm,nim,dynamo}.md); the examples/ directory itself contains only ONE serving .dstack.yml (examples/models/gpt-oss/amd/120b.dstack.yml). All examples are plain `type: service` configs using image + commands + port + model + env + volumes + resources.gpu; none use spot_policy. A preset-like mechanism ALREADY EXISTS: a "UI templates" system (core/models/templates.py UITemplate, server/services/templates.py, routers/templates.py) that pulls template YAMLs from a git repo (DSTACK_SERVER_TEMPLATES_REPO or per-project templates_repo) with an embedded `configuration: Dict[str, Any]` — a strong precedent/reuse candidate for endpoint presets. No "preset" or "recipe" concept exists anywhere in src/. + +## Key files +- mkdocs/docs/examples/inference/vllm.md — — vLLM service YAML (NVIDIA + AMD tabs), Qwen/Qwen3.6-27B on vllm/vllm-openai:v0.19.1 +- mkdocs/docs/examples/inference/sglang.md — — SGLang YAML incl. replica-groups PD-disaggregation example with `replicas: [{count, commands, resources, router|scaling}]` +- mkdocs/docs/examples/inference/nim.md — — NIM YAML: registry_auth with NGC_API_KEY, no commands (image entrypoint), gpu: H100:80GB:8 +- mkdocs/docs/examples/inference/trtllm.md — — TensorRT-LLM YAML: trtllm-serve, gpu: H100:8 +- mkdocs/docs/examples/inference/dynamo.md — — Dynamo PD-disaggregation with replica groups + scaling (metric: rps) +- examples/models/gpt-oss/amd/120b.dstack.yml — — Only real serving .dstack.yml file in examples/; vLLM on MI300X:8, env includes HF_TOKEN +- src/dstack/_internal/core/models/templates.py — UITemplate, AnyUITemplateParameter — Existing template model: type: template, name, title, description, parameters[], configuration: Dict[str,Any] +- src/dstack/_internal/server/services/templates.py — list_templates(project), _fetch_templates_repo, TEMPLATES_DIR_NAME='.dstack/templates' — Clones/pulls a git repo into SERVER_DATA_DIR_PATH/templates-repos/, parses YAML templates, TTLCache 180s +- src/dstack/_internal/server/routers/templates.py — POST /api/project/{project_name}/templates/list — UI-facing list endpoint, ProjectMember permission +- src/dstack/_internal/server/settings.py — SERVER_TEMPLATES_REPO (line 145) — env var DSTACK_SERVER_TEMPLATES_REPO; per-project override via ProjectModel.templates_repo +- mkdocs.yml — — nav lines 329-334 index Inference examples (SGLang, Dynamo, vLLM, NIM, TensorRT-LLM); many redirects show examples were moved from examples/ into docs + +## Details +## 1. Where LLM serving examples live + +The `examples/` directory was heavily trimmed. Current top-level dirs: `distributed-training, llms, misc, models, plugins, server-deployment, single-node-training`. The ONLY serving `.dstack.yml` on disk is `examples/models/gpt-oss/amd/120b.dstack.yml` (vLLM/ROCm). `examples/llms/deepseek/trl/*` are training tasks, not services. There are no per-example README.md files for inference in examples/ (READMEs exist only for misc/, plugins/, qlora, cloudformation). + +The canonical serving examples are YAML blocks embedded in markdown under `mkdocs/docs/examples/inference/`: +- `vllm.md` — vLLM (NVIDIA H100:4 + AMD MI300X:4) +- `sglang.md` — SGLang (incl. replica-groups PD disaggregation) +- `trtllm.md` — TensorRT-LLM +- `nim.md` — NVIDIA NIM +- `dynamo.md` — Dynamo (PD disaggregation, autoscaling) +Also `mkdocs/docs/examples/models/{deepseek-v4.md, qwen36.md}` and `mkdocs/docs/examples/accelerators/*.md`. + +Indexing: `mkdocs.yml` nav lines 316-341 (Examples > Training/Clusters/Inference/Models/Accelerators) plus a hand-written card grid in `mkdocs/docs/examples.md`. No metadata files. There is no TGI service example anymore (TGI appears only in the proxy model-client `src/dstack/_internal/proxy/lib/services/model_proxy/clients/tgi.py`, `format: tgi` in the model spec). + +## 2. Representative full YAMLs (verbatim from repo) + +vLLM (mkdocs/docs/examples/inference/vllm.md, NVIDIA tab): +```yaml +type: service +name: qwen36 +image: vllm/vllm-openai:v0.19.1 +commands: + - | + vllm serve Qwen/Qwen3.6-27B \ + --host 0.0.0.0 \ + --port 8000 \ + --tensor-parallel-size $DSTACK_GPUS_NUM \ + --max-model-len 262144 \ + --reasoning-parser qwen3 +port: 8000 +model: Qwen/Qwen3.6-27B +volumes: + - instance_path: /root/.cache + path: /root/.cache + optional: true +resources: + shm_size: 16GB + gpu: H100:4 +``` +AMD variant differs only in image (`vllm/vllm-openai-rocm:v0.19.1`) and resources (`cpu: 52..`, `memory: 896GB..`, `disk: 450GB..`, `gpu: MI300X:4`). + +SGLang (sglang.md, NVIDIA): same shape with `image: lmsysorg/sglang:v0.5.10.post1`, `commands: sglang serve --model-path Qwen/Qwen3.6-27B --tp $DSTACK_GPUS_NUM ...`, `port: 30000`, `model: Qwen/Qwen3.6-27B`, same cache volume, `resources: {shm_size: 16GB, gpu: H100:4}`. + +NIM (nim.md) — no commands, uses registry_auth + env passthrough: +```yaml +type: service +name: nemotron120 +image: nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:1.8.0 +env: + - NGC_API_KEY +registry_auth: + username: $oauthtoken + password: ${{ env.NGC_API_KEY }} +port: 8000 +model: nvidia/nemotron-3-super-120b-a12b +volumes: + - instance_path: /root/.cache/nim + path: /opt/nim/.cache + optional: true +resources: + cpu: x86:96.. + memory: 512GB.. + shm_size: 16GB + disk: 500GB.. + gpu: H100:80GB:8 +``` + +Real on-disk example `examples/models/gpt-oss/amd/120b.dstack.yml` uses `env: [HF_TOKEN, MODEL=openai/gpt-oss-120b, VLLM_ROCM_USE_AITER=1, ...]` (HF_TOKEN as passthrough), short-form volume `- /root/.cache/huggingface:/root/.cache/huggingface`, `gpu: MI300X:8`. + +Fields observed across all serving examples: `type: service`, `name`, `image`, `commands` (multiline `|` blocks), `port`, `model` (plain HF model-id string), `env` (list; passthrough names like `HF_TOKEN`/`NGC_API_KEY` and `KEY=value` pairs), `volumes` (instance_path/path/optional or short form), `resources` (`gpu: :`, `gpu: H100:80GB:8` with mem, ranges `cpu: 96..`, `memory: 512GB..`, `disk:`, `shm_size:`), `registry_auth`. Advanced: `replicas` as a LIST of replica groups `{count: 1 or 1..4, commands, resources, scaling: {metric: rps, target: 3}, router: {type: sglang}}` (sglang.md line ~155, dynamo.md line 27). `spot_policy` appears in NO example (only in concepts/services.md docs, values `spot|on-demand|auto`). Simple scalar `replicas: N` + `scaling` also documented in sglang/dynamo autoscaling notes. + +## 3. Existing preset/recipe/template machinery + +- "preset"/"recipe": do not exist as concepts anywhere in src/ (grep hits for those words are incidental — e.g. nebius "preset" is a cloud API field, runpod/cloudrift "template" is provider API terminology). +- "template" DOES exist as a real feature: **UI Templates** (added Mar 2026, migration `03_06_1200_a13f5b55af01_add_projectmodel_templates_repo.py`). + - Model: `UITemplate` in `src/dstack/_internal/core/models/templates.py:59` — `{type: "template", name, title, description, parameters: List[AnyUITemplateParameter], configuration: Dict[str, Any]}` where `configuration` is a raw dstack run configuration dict. Parameter types: name, ide, resources, python_or_docker, repo, working_dir, env (with title/name/value). + - Service: `src/dstack/_internal/server/services/templates.py` — `async list_templates(project) -> List[UITemplate]`; clones a git repo (`project.templates_repo` or `settings.SERVER_TEMPLATES_REPO` = env `DSTACK_SERVER_TEMPLATES_REPO`, settings.py:145) into `SERVER_DATA_DIR_PATH/templates-repos/`, reads YAML files from `.dstack/templates` dir in the repo, TTL-cached 180s, run in thread via `run_async`. + - Router: `src/dstack/_internal/server/routers/templates.py` — `POST /api/project/{project_name}/templates/list`, `ProjectMember()` permission, registered in `server/app.py`. + - DB: `ProjectModel.templates_repo` column in `server/models.py`. + - These templates are consumed by the UI only; there is no server-side "instantiate template into a run" logic in src/. + +## 4. Preset sketch grounded in real syntax + +A preset could be a stored service config keyed by model + accelerator, e.g.: +```yaml +# preset for Qwen/Qwen3.6-27B on NVIDIA +model: Qwen/Qwen3.6-27B +service: # verbatim dstack service configuration (like UITemplate.configuration) + type: service + image: vllm/vllm-openai:v0.19.1 + commands: + - | + vllm serve Qwen/Qwen3.6-27B --host 0.0.0.0 --port 8000 \ + --tensor-parallel-size $DSTACK_GPUS_NUM --max-model-len 262144 + port: 8000 + model: Qwen/Qwen3.6-27B + env: + - HF_TOKEN # passthrough, merged with endpoint's env + volumes: + - instance_path: /root/.cache + path: /root/.cache + optional: true + resources: + shm_size: 16GB + gpu: H100:4 # matching key vs fleet GPUs +``` +Every field above is verified real service syntax. The existing UITemplate pattern (raw `configuration: Dict[str, Any]` validated later against the run-configuration parser) is the natural precedent for storing the embedded service config. + +## Gotchas +1) Do NOT assume examples/ contains vLLM/SGLang/TGI .dstack.yml files — it doesn't; the serving YAMLs live only inside mkdocs markdown, so a preset library cannot be built by globbing examples/. 2) There is no TGI serving example anymore (TGI survives only as a proxy model `format`). 3) `replicas` in current examples is a LIST of replica-group objects (count/commands/resources/scaling/router), not just an int — a preset schema must allow both. 4) `model:` in service YAML is a plain string (HF model id); older `model: {type/name/format}` object form was not seen in these examples. 5) No example uses spot_policy; don't copy it into presets as if idiomatic. 6) An existing "templates" feature (UITemplate + git-repo fetch + /templates/list API) already occupies the template namespace — the endpoint plan should either reuse it or deliberately name presets differently to avoid collision. 7) GPU spec strings like `H100:80GB:8` and range syntax `cpu: 96..` / `memory: 512GB..` are valid and used; presets should preserve exact string forms. diff --git a/endpoint-implementation-research/external-recipes.md b/endpoint-implementation-research/external-recipes.md new file mode 100644 index 0000000000..42592b3e78 --- /dev/null +++ b/endpoint-implementation-research/external-recipes.md @@ -0,0 +1,85 @@ +# external-recipes + +## Summary +Both projects exist and are Apache-2.0 licensed, so an embedded dstack agent can fetch, vendor, or redistribute their content with attribution. The SGLang Cookbook's canonical home is now docs.sglang.io/cookbook (source: sgl-project/sglang repo under docs_new/cookbook, MDX files); the old sgl-cookbook GitHub repo was archived read-only on 2026-06-11 and cookbook.sglang.io is legacy. vLLM recipes live at github.com/vllm-project/recipes and render at recipes.vllm.ai, which exposes a purpose-built machine-readable JSON API (/models.json index + per-model JSON with exact vllm serve commands, docker commands, and per-GPU hardware configs) — ideal for runtime WebFetch by an agent. SGLang has an llms.txt full-doc index at docs.sglang.io/llms.txt and serves page source by appending .md, but pages are MDX mixing static serve commands/hardware tables with JSX config-generator components, so parsing is messier; for SGLang, vendoring/sparse-cloning docs_new/cookbook or the archived repo's markdown is more robust. + +## Key files +- https://recipes.vllm.ai/models.json — — EXTERNAL URL (no local files in this research). Verified machine-readable index of ~500 model entries; fields per entry: hf_id, title, provider, url, json (path to per-model JSON), optional derived_from. +- https://recipes.vllm.ai/Qwen/Qwen3-32B.json — — EXTERNAL URL. Verified per-model JSON: meta (title/description/hardware verification), model (parameter_count, context_length, min vLLM version), recommended_command (exact vllm serve string per hardware), docker_command (vllm/vllm-openai:latest), by_hardware map (H200/H100/B200/TPU), variants (BF16/FP8/AWQ), guide section. +- https://github.com/vllm-project/recipes — — EXTERNAL URL. Canonical vLLM recipes repo, Apache-2.0 (LICENSE verified raw). New format: structured YAML at models//.yaml mirroring HuggingFace paths; legacy markdown per org (Qwen/Qwen3.5.md, GLM/GLM5.md, Google/Gemma4.md) still present but being migrated. JSON API rebuilt via scripts/build-recipes-api.mjs. +- https://raw.githubusercontent.com/vllm-project/recipes/main/models/deepseek-ai/DeepSeek-V4-Pro.yaml — — EXTERNAL URL. Verified concrete YAML recipe: model_id deepseek-ai/DeepSeek-V4-Pro (1.6T MoE/49B active, 1,048,576 ctx), verified hardware matrix (H200/B200/B300/GB200/GB300/MI355X; MI300X unsupported), base args --trust-remote-code --kv-cache-dtype fp8 --block-size 256, variants (default FP4+FP8, nvfp4, dspark; 960 GB VRAM min), parallelism strategies (TP/TEP/DEP/multi-node/PD), per-hardware overrides, AMD docker image vllm/vllm-openai-rocm:nightly. +- https://docs.sglang.io/llms.txt — — EXTERNAL URL. Verified: full documentation index including every cookbook recipe; recipe URL pattern https://docs.sglang.io/cookbook/autoregressive//.md (appending .md returns raw MDX source). +- https://docs.sglang.io/cookbook/autoregressive/intro — — EXTERNAL URL. Canonical SGLang Cookbook location. 23 vendor families listed (Qwen/Qwen3.6, DeepSeek/DeepSeek-V4, Llama/Llama3.3-70B, GLM/GLM-5.2, Moonshotai/Kimi-K2.7-Code, OpenAI/GPT-OSS, MiniMax, NVIDIA, Mistral, etc.); also a diffusion section and benchmark pages. +- https://github.com/sgl-project/sglang/tree/main/docs_new — — EXTERNAL URL. Current cookbook SOURCE after migration: cookbook/autoregressive//.mdx (Mintlify MDX + docs.json nav). sglang repo LICENSE verified Apache-2.0 ('Copyright 2023-2024 SGLang Team'). Sparse-checkout of docs_new/cookbook is the vendoring target. +- https://github.com/sgl-project/sgl-cookbook — — EXTERNAL URL. Old cookbook repo — ARCHIVED read-only 2026-06-11, Apache-2.0. Still useful frozen snapshot: plain markdown at docs/autoregressive//.md plus structured YAML configs at data/models/src|generated/v/.yaml (e.g. deepseek.yaml, qwen.yaml) feeding interactive config generators. +- https://raw.githubusercontent.com/sgl-project/sgl-cookbook/main/docs/autoregressive/DeepSeek/DeepSeek-V3_2.md — — EXTERNAL URL. Verified concrete SGLang recipe example: exact 'sglang serve --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --host 0.0.0.0 --port 30000' commands (plus reasoning-parser/tool-call variants), hardware guidance (8x B200 TP=8; 16x H800 TP=16 across 2 nodes; H200/B200/MI355X supported), sections: Introduction, Installation, Deployment, Invocation, Benchmark. + +## Details +## 1. SGLang Cookbook + +**Canonical source (verified):** https://docs.sglang.io/cookbook/intro — the cookbook migrated into the main SGLang docs. Source files live in the main repo at `github.com/sgl-project/sglang/tree/main/docs_new/cookbook` as **Mintlify MDX** (e.g. `cookbook/autoregressive/Qwen/Qwen3.5.mdx`). The standalone repo `github.com/sgl-project/sgl-cookbook` (previously rendered at cookbook.sglang.io) was **archived read-only on 2026-06-11**; its README points contributors to `sglang/docs_new`. I did not fetch cookbook.sglang.io itself, so whether it still serves or redirects is unverified. + +**Structure:** `cookbook/autoregressive//` and `cookbook/diffusion/...`, plus benchmark pages. 23 vendor families verified in the autoregressive index, including `DeepSeek/DeepSeek-V4`, `Qwen/Qwen3.6`, `Llama/Llama3.3-70B`, `GLM/GLM-5.2`, `Moonshotai/Kimi-K2.7-Code`, `OpenAI/GPT-OSS`, `MiniMax/MiniMax-M3`, `NVIDIA/Nemotron3-Ultra`, `Mistral/Ministral-3`. The llms.txt index also lists older recipes retained after migration (DeepSeek-R1/V3/V3.1/V3.2, Qwen2.5-VL/Qwen3/Qwen3-Coder/Qwen3.5, Kimi-K2/K2.5/K2.6/Kimi-Linear). + +**Per-recipe content (verified on two recipes):** +- Exact serve commands. From the DeepSeek-V3.2 recipe (fetched raw from the archived repo): + ``` + sglang serve \ + --model deepseek-ai/DeepSeek-V3.2-Exp \ + --tool-call-parser deepseekv31 \ + --reasoning-parser deepseek-v3 \ + --chat-template ./examples/chat_template/tool_chat_template_deepseekv32.jinja \ + --tp 8 --host 0.0.0.0 --port 30000 + ``` + with hardware guidance: 8x B200 (TP=8) or 16x H800 (TP=16, 2 nodes); supported platforms H200/B200/MI355X. +- Docker image tags. From the current Qwen3.5 MDX: `lmsysorg/sglang:latest` (NVIDIA), `lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi30x-20260604` (MI300X/MI325X), `...mi35x-...` (MI355X); AMD commands use env prefixes like `SGLANG_USE_AITER=1 ... python3 -m sglang.launch_server`. +- GPU sizing tables: e.g. Qwen3.5-397B-A17B — "H100 (80GB) requires tp=16 (2 nodes)" for BF16 vs "tp=8" for FP8; a table maps each GPU to memory + TP across precisions. +- Benchmarks (TTFT/TPOT/tokens-per-sec at multiple concurrencies) and tuning sections. + +**Format caveat (verified):** current pages are MDX — ~90% static text (commands, tables, docker pulls are literal) plus an embedded interactive JSX component (e.g. ``, `Playground` with `dockerImages` config objects). Some pages (DeepSeek-V4) are more JSX-heavy, with commands generated dynamically from embedded config objects rather than written as static shell blocks. The archived sgl-cookbook additionally has structured YAML per model at `data/models/src/v/.yaml` (frozen as of June 2026) — the closest thing SGLang has to machine-readable recipes, but no longer maintained there; whether equivalent YAML lives in `docs_new` is **unverified**. + +## 2. vLLM recipes + +**Canonical source (verified):** https://github.com/vllm-project/recipes, rendered at https://recipes.vllm.ai (144 recipes across 40+ providers, per the site). + +**Structure (verified):** two generations coexist: +- Legacy markdown: top-level org dirs (`Qwen/Qwen3.5.md`, `GLM/GLM5.md`, `Google/Gemma4.md`, ...) — being migrated. +- New structured YAML: `models//.yaml`, path mirrors HuggingFace so `recipes.vllm.ai//` matches `huggingface.co//`. Validated/compiled to a JSON API via `node scripts/build-recipes-api.mjs`; CONTRIBUTING says static JSON is exported under `public/` explicitly "for agents/tools to consume" (that quote from a search snippet; the working endpoints below corroborate it). + +**Concrete example 1 (YAML, verified raw fetch):** `models/deepseek-ai/DeepSeek-V4-Pro.yaml` — model_id `deepseek-ai/DeepSeek-V4-Pro` (1.6T total / 49B active MoE, 1,048,576-token context); verified-hardware matrix H200/B200/B300/GB200/GB300/MI355X (MI300X/MI325X marked unsupported); base args `--trust-remote-code --kv-cache-dtype fp8 --block-size 256`; variants default(FP4+FP8)/nvfp4/dspark, each min ~960 GB VRAM; parallelism strategies (default `single_node_tep`, plus TP/DEP/multi-node/PD-cluster); per-hardware overrides (Hopper: max-model-len capped at 800k; Blackwell: `--moe-backend deep_gemm_mega_moe`; AMD: `--distributed-executor-backend mp`, docker `vllm/vllm-openai-rocm:nightly`). + +**Concrete example 2 (rendered JSON, verified):** `https://recipes.vllm.ai/Qwen/Qwen3-32B.json` — includes `"recommended_command": {"hardware": "h200", "command": "vllm serve Qwen/Qwen3-32B \\\n --tensor-parallel-size 1 \\\n --reasoning-parser qwen3"}`, a full `docker_command` using image `vllm/vllm-openai:latest`, `by_hardware` configs (H200/H100/B200/TPU), quantization `variants` (BF16/FP8/AWQ), model metadata (32B params, 40960 ctx, min vLLM version), and a prose `guide`. + +## 3. Licenses (both verified from LICENSE files) + +- `vllm-project/recipes`: **Apache License 2.0** (raw LICENSE fetched). +- `sgl-project/sglang` (hosts docs_new/cookbook): **Apache License 2.0**, "Copyright 2023-2024 SGLang Team" (raw LICENSE fetched). The archived `sgl-cookbook` repo is also Apache-2.0 (per its repo page). + +Implication: runtime fetching, caching, vendoring, and redistribution of recipe content by a dstack-embedded agent are all permitted; if vendoring, retain the Apache-2.0 license text and attribution notices. (Note: the licenses cover the recipe text/config, not the model weights they describe.) + +## 4. Runtime WebFetch vs clone/vendor + +**vLLM — fetch at runtime, it's designed for it:** +- Index: `GET https://recipes.vllm.ai/models.json` (verified; ~500 entries incl. quantized variants; fields hf_id/title/provider/url/json/derived_from). +- Per model: `GET https://recipes.vllm.ai//.json` (verified) — already contains the exact serve command per hardware; no MD parsing needed. +- Fallback/stable raw: `https://raw.githubusercontent.com/vllm-project/recipes/main/models//.yaml` (verified working). +- Churn risk: legacy `*/.md` paths are mid-migration — don't build against those. + +**SGLang — fetchable but messier; vendoring is safer:** +- Discoverable at runtime via `https://docs.sglang.io/llms.txt` (verified index) and per-page raw source by appending `.md` (verified: returns MDX source, not clean markdown). +- Risk: content is MDX with JSX components (some recipes like DeepSeek-V4 keep docker images/args inside JSX config objects, not shell blocks), and the cookbook already changed homes once within a month (cookbook.sglang.io → docs.sglang.io), so URL churn risk is real. +- Vendoring option: sparse-checkout of `sglang/docs_new/cookbook` (current, MDX) or the frozen archived `sgl-cookbook` (plain md + structured `data/models/src/*.yaml`, but stale post-2026-06-11). + +## 5. Machine-readable indexes + +- **vLLM:** `recipes.vllm.ai/models.json` + per-model `//.json` (both verified). No llms.txt — `https://recipes.vllm.ai/llms.txt` returns **404** (verified). +- **SGLang:** `https://docs.sglang.io/llms.txt` (verified; full doc/cookbook index with .md URLs — Mintlify-style). No JSON recipe API found for SGLang; the archived repo's `data/models/src|generated/v*/{model}.yaml` config-generator data is the nearest equivalent (frozen). Whether docs.sglang.io exposes llms-full.txt or an equivalent JSON is **unverified** (not checked). + +## Unverified items (explicit) +- Current behavior of cookbook.sglang.io (redirect vs stale mirror) — not fetched. +- Whether every docs.sglang.io page reliably serves source via the `.md` suffix (verified only for DeepSeek-V4.md). +- The `public/` static-JSON claim in vLLM CONTRIBUTING.md was seen only in a search snippet (endpoints themselves verified live). +- Post-knowledge-cutoff model names (DeepSeek-V4, Qwen3.5/3.6, GLM-5.2, Kimi-K2.7-Code, etc.) are reported exactly as returned by the fetched pages. + +## Gotchas +1) The "SGLang Cookbook" GitHub repo (sgl-project/sgl-cookbook) is ARCHIVED (read-only since 2026-06-11) — do not plan around it as the live source; the live source is sgl-project/sglang:docs_new/cookbook and the live site is docs.sglang.io/cookbook (NOT cookbook.sglang.io). 2) SGLang recipe pages are MDX, not plain markdown: appending .md to a docs.sglang.io URL returns raw MDX where some recipes (e.g. DeepSeek-V4) embed docker images and serve-arg matrices inside JSX Playground/ConfigGenerator components rather than static shell blocks — a naive markdown-reading agent will miss or misparse those; older/most recipes (e.g. Qwen3.5) are ~90% static and fine. 3) SGLang cookbook URLs churned once already (repo + domain moved in June 2026), so hardcoded deep links are fragile; resolve via docs.sglang.io/llms.txt at runtime. 4) vLLM recipes repo is mid-migration from per-org markdown to models//.yaml — build against recipes.vllm.ai/models.json + per-model JSON (or raw YAML on GitHub), never the legacy .md paths. 5) recipes.vllm.ai has NO llms.txt (404) — its discovery entry point is /models.json; conversely SGLang has llms.txt but NO JSON API. An agent needs two different consumption strategies. 6) Apache-2.0 on both covers the recipe text only, not the model weights the recipes deploy; vendored copies must retain LICENSE/attribution. 7) vLLM's models.json includes ~500 entries but many are quantized variants linked via derived_from — dedupe on that field when presenting model choices. diff --git a/endpoint-implementation-research/resource-lifecycle-blueprint.md b/endpoint-implementation-research/resource-lifecycle-blueprint.md new file mode 100644 index 0000000000..b5c9b5dde6 --- /dev/null +++ b/endpoint-implementation-research/resource-lifecycle-blueprint.md @@ -0,0 +1,201 @@ +# resource-lifecycle-blueprint + +## Summary +Gateways and volumes are the two existing non-run resources with a full provisioning lifecycle, and they share one architecture: a SQLAlchemy model with PipelineModelMixin (lock_expires_at/lock_token/lock_owner) + status/status_message/last_processed_at/to_be_deleted columns; a status enum in core/models with SUBMITTED -> (PROVISIONING) -> RUNNING/ACTIVE | FAILED; a service module with create_*/delete_*/list_* functions raising ServerClientError subclasses; a Pipeline subclass (Fetcher/Worker/Heartbeater) in server/background/pipeline_tasks registered in PipelineManager; a FastAPI router with ProjectAdmin (gateways) or ProjectMember (volumes) permission deps; a BaseApplyConfigurator subclass wired into apply_configurators_mapping for `dstack apply`/`dstack delete`; and an APIClientGroup on APIClient. Deletion is asynchronous in both cases: API handlers only set to_be_deleted=True, and the pipeline worker performs backend cleanup and then hard-deletes the row (gateway) or soft-deletes with deleted=True/deleted_at (volume). All processing is multi-replica safe via SELECT ... FOR UPDATE SKIP LOCKED + lock_token/lock_expires_at leases with a heartbeat task, and update-after-processing is written via UPDATE ... WHERE id=... AND lock_token=... so lost locks are detected. I verified every symbol below by reading the code. + +## Key files +- src/dstack/_internal/server/models.py — PipelineModelMixin (L204), GatewayModel (L600), GatewayComputeModel (L656), VolumeModel (L951), VolumeAttachmentModel (L1001) — DB models. UUID pk (UUIDType(binary=False), default=uuid.uuid4), project FK with ondelete CASCADE, status stored as EnumAsString(StatusEnum, 100), Text status_message, NaiveDateTime last_processed_at, Boolean to_be_deleted (server_default=false()), JSON blobs stored as Text (configuration, volume_provisioning_data). Partial index ix__pipeline_fetch_q on last_processed_at.asc() filtered by deleted==false. +- src/dstack/_internal/core/models/gateways.py — GatewayStatus (L17), GatewayReplicaStatus (L24), GatewayConfiguration (L59, type: Literal["gateway"]), GatewaySpec (L125), Gateway (L141), GatewayPlan (L176), ApplyGatewayPlanInput (L185) — Core (client-visible) pydantic models. GATEWAY_REPLICAS_DEFAULT=1 at L14. +- src/dstack/_internal/core/models/volumes.py — VolumeStatus (L19), BaseVolumeConfiguration (L36), AnyVolumeConfiguration (L179), VolumeConfiguration (L187), parse_volume_configuration (L191), VolumeSpec (L198), Volume (L233), VolumePlan (L287) — VolumeStatus: SUBMITTED, PROVISIONING (currently unused), ACTIVE, FAILED; finished_statuses() == [FAILED]. +- src/dstack/_internal/core/models/configurations.py — ApplyConfigurationType (L1384), AnyApplyConfiguration (L1393), BaseApplyConfiguration (L1401), parse_apply_configuration (L1424), AnyRunConfiguration (L1366) — Where a new top-level `type: endpoint` must be registered (enum value + union member + discriminator __root__). +- src/dstack/_internal/server/services/volumes.py — create_volume (L259), delete_volumes (L321), list_volumes (L104), get_volume_by_name (L222), volume_model_to_volume (L379), switch_volume_status (L54), emit_volume_status_change_event (L75), generate_volume_name (L457) — The cleanest single-file service blueprint for a new resource type. +- src/dstack/_internal/server/services/gateways/__init__.py — create_gateway (L224), delete_gateways (L335), get_plan (L926), apply_plan (L973), switch_gateway_status (L92), gateway_model_to_gateway (L879), get_gateway_configuration (L847), generate_gateway_name (L617) — Gateway service; includes the plan/apply (in-place update) pattern volumes lack. +- src/dstack/_internal/server/background/pipeline_tasks/base.py — PipelineItem (L34), Pipeline (L67), Heartbeater (L166), Fetcher (L257), Worker (L340), ItemUpdateMap (L403), NOW_PLACEHOLDER (L383), set_unlock_update_map_fields (L410), set_processed_update_map_fields (L416), resolve_now_placeholders (L430), log_lock_token_mismatch (L449), log_lock_token_changed_after_processing (L463) — Framework every new resource pipeline subclasses. Multi-replica safety = FOR UPDATE SKIP LOCKED fetch + lock_token lease + heartbeat renewal + conditional UPDATE on write-back. +- src/dstack/_internal/server/background/pipeline_tasks/volumes.py — VolumePipeline (L58), VolumeFetcher.fetch (L136), VolumeWorker.process (L213), _process_submitted_volume (L307), _process_to_be_deleted_volume (L374) — Simplest full pipeline example (~420 lines) — best copy-paste template for an EndpointPipeline. +- src/dstack/_internal/server/background/pipeline_tasks/gateways.py — GatewayPipeline (L59), GatewayFetcher.fetch (L137), GatewayWorker.process (L215), _process_submitted_gateway (L283), _process_provisioning_gateway (L380), _process_to_be_deleted_gateway (L475) — Example with an intermediate PROVISIONING state that polls child resources (gateway_computes) until all RUNNING; hard-deletes the row on deletion. +- src/dstack/_internal/server/background/pipeline_tasks/__init__.py — PipelineManager (L31), PipelineHinter (L80), get_pipeline_manager (L99), start_pipeline_tasks (L106) — New pipelines must be added to the PipelineManager.__init__ builtin list (L35-48). +- src/dstack/_internal/server/services/pipelines.py — PipelineHinterProtocol (L6), get_pipeline_hinter (L22) — FastAPI dependency giving handlers a hinter; create/apply endpoints call pipeline_hinter.hint_fetch(Model.__name__) after commit for low-latency pickup. +- src/dstack/_internal/server/routers/volumes.py — root_router + project_router; list/get/create/delete handlers — All handlers use Depends(ProjectMember()); create/delete return nothing special; ResourceNotExistsError raised for missing get. +- src/dstack/_internal/server/routers/gateways.py — router (prefix /api/project/{project_name}/gateways); get_plan (L74) + apply_plan (L96) use ProjectAdmin(); list uses ProjectMemberOrPublicAccess(); get uses Authenticated()+Project()+check_can_access_gateway — Registered in app.py L257-259 via app.include_router(). +- src/dstack/_internal/server/schemas/volumes.py — ListVolumesRequest, GetVolumeRequest, CreateVolumeRequest, DeleteVolumesRequest — Request body models; gateways version adds GetGatewayPlanRequest/ApplyGatewayPlanRequest. +- src/dstack/_internal/server/security/permissions.py — Authenticated (L37), GlobalAdmin (L49), ProjectAdmin (L63), ProjectManager (L84), ProjectMember (L112), ProjectMemberOrPublicAccess (L123), check_can_access_gateway (L315) — Exact dependency names. ProjectAdmin/ProjectMember __call__ returns Tuple[UserModel, ProjectModel]. +- src/dstack/_internal/cli/services/configurators/__init__.py — apply_configurators_mapping (L27), get_apply_configurator_class (L52), load_apply_configuration (L62) — New configurator class must be appended to the mapping list. +- src/dstack/_internal/cli/services/configurators/volume.py — VolumeConfigurator (TYPE=ApplyConfigurationType.VOLUME), apply_configuration, delete_configuration, register_args, apply_args — Client-side plan, delete-then-recreate on change, provisioning polling loop until ACTIVE/FAILED, exit(1) on FAILED. +- src/dstack/_internal/cli/services/configurators/gateway.py — GatewayConfigurator; uses server-side gateways.get_plan/apply_plan with legacy fallback — Polls until RUNNING/FAILED via _finished_provisioning (L284). +- src/dstack/api/server/__init__.py — APIClient.gateways property (L125) -> GatewaysAPIClient, APIClient.volumes (L129) -> VolumesAPIClient — Low-level client; per-resource group files _gateways.py/_volumes.py subclass APIClientGroup. +- src/dstack/_internal/server/services/events.py — emit (L171), Target.from_model (L89), SystemActor (L34), UserActor (L42), AnyActor (L61) — Target.from_model raises ValueError for unknown model types — a new resource model must be added to its Union + isinstance chain, plus EventTargetType in core/models/events.py (L12). +- src/dstack/_internal/server/services/locking.py — get_locker (L175), ResourceLocker.get_lockset (L37), ResourceLocker.lock_ctx (L46), string_to_lock_id (L121) — In-process/pg-advisory locking used for name-uniqueness during create and row lock acquisition during delete. +- src/dstack/_internal/server/testing/common.py — create_gateway (L639), create_gateway_compute (L683), create_volume (L1069) — Test factories; tests live in src/tests/_internal/server/background/pipeline_tasks/test_{volumes,gateways}.py and src/tests/_internal/server/routers/test_{volumes... (test_gateways.py exists; no test_volumes.py in routers dir listing — volumes router tests may be elsewhere). + +## Details +## 1. DB models (src/dstack/_internal/server/models.py) + +### PipelineModelMixin (models.py:204-207) +```python +class PipelineModelMixin: + lock_expires_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + lock_token: Mapped[Optional[uuid.UUID]] = mapped_column(UUIDType(binary=False)) + lock_owner: Mapped[Optional[str]] = mapped_column(String(100)) +``` +Every pipeline-processed model inherits this (GatewayModel, GatewayComputeModel, VolumeModel, FleetModel, PlacementGroupModel, ...). + +### GatewayModel (models.py:600-653), `__tablename__ = "gateways"` +- `id: Mapped[uuid.UUID] = mapped_column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4)` +- `name: String(100)`, `region: String(100)`, `wildcard_domain: Optional[String(100)]` +- `configuration: Mapped[Optional[str]] = mapped_column(Text)` — JSON of GatewayConfiguration (Optional only for pre-0.18.2 compat) +- `created_at: NaiveDateTime, default=get_current_datetime` +- `status: Mapped[GatewayStatus] = mapped_column(EnumAsString(GatewayStatus, 100))` (models.py:614) +- `status_message: Mapped[Optional[str]] = mapped_column(Text)` (615) +- `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime)` (616) +- `to_be_deleted: Mapped[bool] = mapped_column(Boolean, server_default=false())` (617) +- `project_id: ForeignKey("projects.id", ondelete="CASCADE")` + `project` relationship (624-625); `backend_id` FK likewise (626) +- `__table_args__ = (UniqueConstraint("project_id", "name", name="uq_gateways_project_id_name"),)` (651) +- NO `deleted` flag — gateways are hard-deleted (comment at 653: "TODO: Add pipeline index ... if gateways become soft-deleted"). + +### VolumeModel (models.py:951-998), `__tablename__ = "volumes"` +- `id` UUID pk as above; `name: String(100)` +- `user_id: ForeignKey("users.id", ondelete="CASCADE")` + `user` relationship (959-960) +- `project_id` FK CASCADE + `project` (962-963) +- `created_at`, `last_processed_at: NaiveDateTime, default=get_current_datetime` (965-968), `last_job_processed_at: Optional` (969) +- `deleted: Boolean, default=False` (973), `deleted_at: Optional[NaiveDateTime]` (974), `to_be_deleted: Boolean, server_default=false()` (975) — soft delete +- `status: mapped_column(EnumAsString(VolumeStatus, 100), index=True)` (977) with docstring "`status` must be changed only via `switch_volume_status()`" and `status_message: Text` (979) +- JSON-as-Text columns: `configuration` (981, required), `volume_provisioning_data` (982, optional) +- `attachments: relationship(VolumeAttachmentModel)` (986) +- Partial fetch index (991-998): +```python +Index("ix_volumes_pipeline_fetch_q", last_processed_at.asc(), + postgresql_where=deleted == false(), sqlite_where=deleted == false()) +``` + +### GatewayComputeModel (models.py:656-733) — child resource per gateway replica +Has its own `status: GatewayReplicaStatus`, `status_message`, `last_processed_at`, `deleted`, `ssh_private_key/ssh_public_key` (Text), `backend_data: Text`, and its own pipeline fetch index `ix_gateway_computes_pipeline_fetch_q` (727-732). + +## 2. Status enums and lifecycle + +`src/dstack/_internal/core/models/gateways.py:17-29`: +```python +class GatewayStatus(str, Enum): + SUBMITTED = "submitted"; PROVISIONING = "provisioning"; RUNNING = "running"; FAILED = "failed" +class GatewayReplicaStatus(str, Enum): + SUBMITTED, PROVISIONING, RUNNING, TERMINATING, TERMINATED +``` +`src/dstack/_internal/core/models/volumes.py:19-33`: +```python +class VolumeStatus(str, Enum): + SUBMITTED, PROVISIONING (unused), ACTIVE, FAILED + def is_active(self); @classmethod finished_statuses() -> [FAILED] +``` + +Lifecycle, volumes: API `create_volume` inserts row with `status=SUBMITTED` -> VolumeWorker `_process_submitted_volume` calls backend `compute.create_volume`/`register_volume` -> writes `status=ACTIVE, volume_provisioning_data=vpd.json()` or `status=FAILED, status_message=...`. There is no PROVISIONING step in practice. + +Lifecycle, gateways: `create_gateway` inserts `status=SUBMITTED` -> `_process_submitted_gateway` resolves backend, creates N GatewayComputeModel rows (SUBMITTED), sets gateway `status=PROVISIONING` -> the separate GatewayReplicaPipeline (`background/pipeline_tasks/gateway_replicas.py`) provisions each replica (`compute.create_gateway(compute_configuration)` at gateway_replicas.py:427, then connect + `gateways_services.configure_gateway(connection)` at :537) -> gateway pipeline `_process_provisioning_gateway` (gateways.py:380-401) polls replica statuses: any TERMINATING/TERMINATED => `FAILED` with `status_message="Failed to provision gateway replica"`; all RUNNING => `RUNNING`; otherwise no-op (stays PROVISIONING, gets re-fetched after min_processing_interval=15s). + +Deletion — asynchronous in both cases, user-facing call only marks the row: +- `delete_volumes` (services/volumes.py:321): selects rows, then under `get_locker(...).lock_ctx(VolumeModel.__tablename__, volumes_ids)` retries 10x to grab rows with `lock_expires_at IS NULL` FOR UPDATE; raises `ServerClientError("Failed to delete volumes: volumes are being processed currently. Try again later.")` if it can't; raises ServerClientError if attachments exist ("Volume is in use"); sets `to_be_deleted=True` + emits "Volume marked for deletion" event; commits. The pipeline `_process_to_be_deleted_volume` (pipeline_tasks/volumes.py:374) calls `compute.delete_volume` (errors logged, not retried) then writes `{deleted: True, deleted_at: NOW_PLACEHOLDER}` — soft delete. +- `delete_gateways` (services/gateways/__init__.py:335): same lock-retry pattern, sets `to_be_deleted=True`. GatewayWorker `_process_to_be_deleted_item` (pipeline_tasks/gateways.py:404) waits until all replica computes are TERMINATED (replica termination is driven by the GatewayReplicaPipeline) and then executes `delete(GatewayModel).where(id==..., lock_token==...)` — hard delete of the row, emitting "Gateway deleted" with `events.SystemActor()`. +- Additionally `process_idle_volumes` (background/scheduled_tasks/idle_volumes.py:25) sets `to_be_deleted = True` (line 92) for volumes idle past `auto_cleanup_duration` — an example of system-initiated deletion. + +## 3. Service modules + +### services/volumes.py +- `async def create_volume(session, project: ProjectModel, user: UserModel, configuration: AnyVolumeConfiguration, pipeline_hinter: PipelineHinterProtocol) -> Volume` (L259). Flow: `apply_plugin_policies(user=user.name, project=project.name, spec=VolumeSpec(configuration=configuration))` -> `_validate_volume_configuration` (raises `ServerClientError` for bad config / unsupported backend; `validate_dstack_resource_name` from `dstack._internal.core.services`) -> name-uniqueness critical section: `lock_namespace = f"volume_names_{project.name}"`; on sqlite `await session.commit()` first, on postgres `SELECT pg_advisory_xact_lock(string_to_lock_id(lock_namespace))`, plus in-process `get_locker(get_db().dialect_name).get_lockset(lock_namespace)`; duplicate name => `raise ResourceExistsError()`; auto-name via `generate_volume_name` (random_names.generate_name loop) -> insert model with `status=VolumeStatus.SUBMITTED, created_at=now, last_processed_at=now` -> `events.emit(...,"Volume created. Status: SUBMITTED", actor=events.UserActor.from_user(user), targets=[events.Target.from_model(volume_model)])` -> commit -> `pipeline_hinter.hint_fetch(VolumeModel.__name__)` (L317). +- `list_volumes(session, user, project_name, only_active, prev_created_at, prev_id, limit, ascending) -> List[Volume]` (L104) — keyset pagination on (created_at, id). +- `volume_model_to_volume(volume_model) -> Volume` (L379) — model->API conversion; JSON parsed via `VolumeConfiguration.__response__.parse_raw(...)`. +- `switch_volume_status(session, volume_model, new_status, actor=events.SystemActor())` (L54) — the only sanctioned in-session status setter, emits status-change event ("Volume status changed OLD -> NEW (message)"). + +### services/gateways/__init__.py +- `create_gateway(session, user, project, configuration: GatewayConfiguration, pipeline_hinter, *, effective_configuration=None) -> Gateway` (L224) — same plugin-policy/lock/name pattern (`lock_namespace = f"gateway_names_{project.name}"`); also resolves backend up-front with `get_project_backend_with_model_by_type_or_error` and handles default-gateway assignment; hints via `pipeline_hinter.hint_fetch(GatewayModel.__name__)` (L291). +- `get_plan(session, project, user, spec: GatewaySpec) -> GatewayPlan` (L926) — computes `ApplyAction.CREATE` vs `UPDATE` using `diff_models` (from `dstack._internal.core.services.diff`) and `_can_update_gateway_in_place` (L1050, checks diff keys against `_CONF_UPDATABLE_FIELDS`). +- `apply_plan(session, user, project, plan: ApplyGatewayPlanInput, force: bool, pipeline_hinter) -> Gateway` (L973) — creates if name unset/not found; rejects if `to_be_deleted` ("is being deleted. Try again later."); optimistic concurrency check unless `force` ("Failed to apply plan. Resource has been changed. Try again or use force apply."); in-place update writes configuration + emits "Gateway updated. Changed fields: ...". + +Error convention (src/dstack/_internal/core/errors.py): `ServerClientError` (L39) is the base 400-class error carrying `msg` and `code: ServerClientErrorCode`; subclasses `ResourceExistsError` (L52, msg "Resource exists"), `ResourceNotExistsError` (L57, "Resource not found"), `BackendNotAvailable` (L67), `GatewayError` (L80). Routers raise them directly; `get_base_api_additional_responses()` + exception handling in `server/utils/routers.py` (get_server_client_error_details L111) turn them into HTTP error bodies. `error_forbidden()/error_not_found()/error_invalid_token()` helpers are also in utils/routers.py (L83-104). + +## 4. Background processing + +Two mechanisms (see docstrings): +- `start_pipeline_tasks()` (background/pipeline_tasks/__init__.py:106): "fetch-workers pipelines based on db + in-memory queues ... tasks that run frequently and need to lock rows for a long time". +- `start_scheduled_tasks()` (background/scheduled_tasks/__init__.py:37): APScheduler `AsyncIOScheduler` for infrequent jobs (`process_gateways_connections` every 15s, `process_idle_volumes` every 60s, etc.). + +Both are started in the FastAPI lifespan in app.py (L178-181) guarded by `settings.SERVER_BACKGROUND_PROCESSING_ENABLED`, and `app.state.pipeline_manager = pipeline_manager` is what `get_pipeline_hinter` (services/pipelines.py:22) reads. Shutdown: `pipeline_manager.shutdown()` then `await pipeline_manager.drain()` (app.py:207-214). + +Pipeline anatomy (base.py): a `Pipeline` owns one `Fetcher`, N `Worker`s (default `workers_num=10`), one `Heartbeater`, and an `asyncio.Queue` sized `workers_num*2.0` max. Constructor defaults on VolumePipeline/GatewayPipeline: `min_processing_interval=15s, lock_timeout=30s, heartbeat_trigger=15s`. Required overrides: `hint_fetch_model_name` property (returns `Model.__name__`), `_heartbeater`, `_fetcher`, `_workers` properties. + +One VolumeFetcher.fetch iteration (pipeline_tasks/volumes.py:136-196), multi-replica safe: +1. In-process lockset for the table + session. +2. `select(VolumeModel).where(or_(status==SUBMITTED, to_be_deleted==True), deleted==False, or_(last_processed_at <= now - min_processing_interval, last_processed_at == created_at), or_(lock_expires_at.is_(None), lock_expires_at < now), or_(lock_owner.is_(None), lock_owner == VolumePipeline.__name__)).order_by(last_processed_at.asc()).limit(limit).with_for_update(skip_locked=True, key_share=True, of=VolumeModel)` with `load_only(...)`. +3. For each row set `lock_expires_at = now + lock_timeout`, `lock_token = uuid4()` (one token per batch), `lock_owner = VolumePipeline.__name__`; commit; return `VolumePipelineItem(__tablename__=..., id, lock_expires_at, lock_token, prev_lock_expired, status, to_be_deleted)` items which are queued and tracked by the Heartbeater (renews lock via conditional UPDATE while item is in flight). + +VolumeWorker.process (L213-226): refetch the row `WHERE id==item.id AND lock_token==item.lock_token` with joinedloads (`_refetch_locked_volume` L229); on mismatch `log_lock_token_mismatch` and drop. Dispatch: `to_be_deleted` -> `_process_to_be_deleted_volume`; `status==SUBMITTED` -> `_process_submitted_volume`. Result is a `_ProcessResult(update_map: _VolumeUpdateMap)` (TypedDict extending `ItemUpdateMap` with `status, status_message, volume_provisioning_data, deleted, deleted_at`). `_apply_process_result` (L249) merges, applies `set_processed_update_map_fields` (last_processed_at=NOW_PLACEHOLDER) + `set_unlock_update_map_fields` (nulls the 3 lock columns), resolves NOW placeholders, and executes `update(VolumeModel).where(id==..., lock_token==...).values(**update_map).returning(id)`; 0 rows => `log_lock_token_changed_after_processing` (state change is abandoned, will be reprocessed); else emits the delete/status-change event in the same transaction. + +Exception handling in workers: `Worker.start` (base.py:352-369) wraps `self.process(item)` in try/except logging "Unexpected exception when processing item" and always untracks from heartbeater — an unhandled exception leaves the row locked until `lock_expires_at` passes, then it is re-fetched. Inside `_process_submitted_volume`, `BackendNotAvailable` -> FAILED/"Backend not available"; `BackendError` -> FAILED with `str(e.args[0])` or `f"Backend error: {repr(e)}"`; bare `Exception` -> FAILED/`f"Unexpected error: {repr(e)}"` (L307-361). Sentry instrumentation via `@sentry_utils.instrument_pipeline_task("VolumeFetcher.fetch")` / `("VolumeWorker.process")`. + +Registration: add pipeline instance to the list in `PipelineManager.__init__` (pipeline_tasks/__init__.py:35-48). `PipelineHinter.register_pipeline` maps `hint_fetch_model_name` -> pipeline so API handlers can `hint_fetch("VolumeModel")`; unknown names just log a warning (L88-93). + +## 5. REST API + +Gateways router (routers/gateways.py): `router = APIRouter(prefix="/api/project/{project_name}/gateways", tags=["gateways"], responses=get_base_api_additional_responses())`. Endpoints: POST `/list` (ProjectMemberOrPublicAccess), `/get` (Authenticated + `Project()` dep from `server/deps.py:10` + `check_can_access_gateway` for cross-project imported gateways), `/get_plan` + `/apply` + `/create`(deprecated) + `/delete` + `/set_default` + `/set_wildcard_domain`(deprecated) — all mutating ones use `Depends(ProjectAdmin())`. `/apply` and `/create` take `pipeline_hinter: Annotated[PipelineHinterProtocol, Depends(get_pipeline_hinter)]`. Responses wrapped in `CustomORJSONResponse`; client-version compatibility patching via `patch_gateway/patch_gateway_plan` from `server/compatibility/gateways.py` keyed on `Depends(get_client_version)`. + +Volumes router (routers/volumes.py): TWO routers — `root_router = APIRouter(prefix="/api/volumes")` (global paginated `/list`, `Depends(Authenticated())`) and `project_router = APIRouter(prefix="/api/project/{project_name}/volumes")` (`/list`, `/get`, `/create`, `/delete` — ALL `Depends(ProjectMember())`, i.e. volumes need only member rights while gateways need admin). Both registered in `app.py` (L257-259): `app.include_router(gateways.router); app.include_router(volumes.root_router); app.include_router(volumes.project_router)`. + +Permission dependency exact names (security/permissions.py): `Authenticated` (L37, returns UserModel), `GlobalAdmin` (L49), `ProjectAdmin` (L63, returns Tuple[UserModel, ProjectModel]; global admins pass), `ProjectManager` (L84), `ProjectMember` (L112), `ProjectMemberOrPublicAccess` (L123), `ProjectManagerOrPublicProject` (L159), `ProjectManagerOrSelfLeave` (L195), `OptionalServiceAccount` (L236), plus `check_can_access_gateway` (L315). + +Schemas: `server/schemas/gateways.py` (CreateGatewayRequest, ListGatewaysRequest, GetGatewayRequest, GetGatewayPlanRequest, ApplyGatewayPlanRequest{plan, force}, DeleteGatewaysRequest{names}, SetDefaultGatewayRequest, SetWildcardDomainRequest); `server/schemas/volumes.py` (ListVolumesRequest{project_name, only_active, prev_created_at, prev_id, limit(<=100), ascending}, GetVolumeRequest{name}, CreateVolumeRequest{configuration: Annotated[AnyVolumeConfiguration, Field(discriminator="backend")]}, DeleteVolumesRequest{names}). All extend `CoreModel` from `core/models/common.py`. + +## 6. CLI + +`dstack apply -f x.yml` (cli/commands/apply.py `ApplyCommand`): `load_apply_configuration` (cli/services/configurators/__init__.py:62) parses YAML with `parse_apply_configuration` (core/models/configurations.py:1424), then `get_apply_configurator_class(configuration.type)` looks up `apply_configurators_mapping` (L27) = {DevEnvironmentConfigurator, TaskConfigurator, ServiceConfigurator, FleetConfigurator, GatewayConfigurator, VolumeConfigurator} keyed by `cls.TYPE: ApplyConfigurationType`. Configurator contract (`BaseApplyConfigurator` in configurators/base.py:20): ClassVar `TYPE`, `__init__(self, api_client: Client)`, abstract `apply_configuration(conf, configuration_path, command_args, configurator_args)` and `delete_configuration(conf, configuration_path, command_args)`, classmethods `get_parser()`/`register_args(parser)`. Flags handled by ApplyCommand itself: `-f/--file`, `-y/--yes`, `--force`, `-d/--detach`, `-v/--verbose`. + +`dstack delete -f x.yml` (cli/commands/delete.py `DeleteCommand`, ALIASES=["destroy"]) — same lookup, calls `configurator.delete_configuration(...)`. + +VolumeConfigurator.apply_configuration behavior: builds `VolumeSpec`, computes plan CLIENT-side (`_get_plan` at configurator/volume.py:174 — comment "TODO: Implement server-side /get_plan"), prints plan table, on config change deletes existing volume and polls `volumes.get` until `ResourceNotExistsError`, then `api.client.volumes.create(...)`, and unless `--detach` polls with `MultiItemStatus` every `LIVE_TABLE_PROVISION_INTERVAL_SECS` until status in [ACTIVE, FAILED]; on FAILED prints status_message and `exit(1)`; on Ctrl-C offers to delete. + +GatewayConfigurator uses server-side `gateways.get_plan`/`apply_plan` (with `MethodNotAllowedError` fallback to legacy create for pre-0.20.27 servers) and polls until RUNNING/FAILED. + +Management commands: `GatewayCommand` (cli/commands/gateway.py, NAME="gateway": subcommands list [default]/delete/update/get) and `VolumeCommand` (cli/commands/volume.py, NAME="volume": _list/_delete). Both registered in cli/main.py (`GatewayCommand.register(subparsers)` L74, `VolumeCommand.register(subparsers)` L86; `ApplyCommand` L67, `DeleteCommand` L69). + +## 7. Python API client (src/dstack/api/...) + +Low-level: `dstack/api/server/__init__.py` `class APIClient` with properties `gateways -> GatewaysAPIClient` (L125) and `volumes -> VolumesAPIClient` (L129). `dstack/api/server/_volumes.py` `VolumesAPIClient(APIClientGroup)` methods: `list(project_name)`, `get(project_name, name)`, `create(project_name, configuration)` (uses `get_create_volume_excludes` from `core/compatibility/volumes.py` to trim the body for old servers), `delete(project_name, names)`. `_gateways.py` `GatewaysAPIClient`: `list(project_name, *, include_imported=False)`, `get`, `get_plan(project_name, spec)`, `apply_plan(project_name, plan, *, force=False)`, `create`, `delete`, `set_default`, `set_wildcard_domain`. Responses parsed with `parse_obj_as(Model.__response__, resp.json())`. + +High-level: `dstack/api/_public/__init__.py` `Client` exposes only `runs`, `repos`, `backends` collections — there is NO high-level gateway/volume collection; the CLI reaches them via `self.api.client.gateways`/`self.api.client.volumes` (the raw `APIClient` behind `Client.client`). + +## 8. Supporting infrastructure verified + +- Events: `services/events.py` `emit(session, message, actor, targets)` (L171, adds to session without committing); `Target.from_model` (L89) accepts a closed Union of models and raises `ValueError(f"Unsupported model type: ...")` otherwise; `EventTargetType` enum in `core/models/events.py:12` (PROJECT/USER/FLEET/INSTANCE/RUN/JOB/VOLUME/GATEWAY/SECRET). A new resource requires: new EventTargetType member + new branch in Target.from_model. +- Plugins: `services/plugins.py` `apply_plugin_policies(user: str, project: str, spec: ApplySpec) -> ApplySpec` (L94); `ApplySpec = TypeVar("ApplySpec", RunSpec, FleetSpec, VolumeSpec, GatewaySpec)` in `dstack/plugins/_models.py:8` — a new spec type must be added to this TypeVar for plugin support. +- Locking: `services/locking.py` `get_locker(dialect_name) -> ResourceLocker` (L175); `get_lockset(namespace)`, `lock_ctx(namespace, keys)` (async ctx), `string_to_lock_id(s) -> int` (L121, for pg advisory locks). +- Backends: `services/backends/__init__.py` `get_project_backend_by_type_or_error` (L419), `get_project_backend_with_model_by_type_or_error` (L383), `check_backend_type_available` (L514). +- Migrations: alembic files under `src/dstack/_internal/server/migrations/versions//MM_DD_HHMM__.py`; pattern uses `op.batch_alter_table` (sqlite-compatible) and `dstack._internal.server.models.NaiveDateTime()`/`sqlalchemy_utils.types.uuid.UUIDType(binary=False)` column types (see 2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py). +- Tests: pipeline tests in `src/tests/_internal/server/background/pipeline_tasks/test_volumes.py`, `test_gateways.py`, `test_gateway_replicas.py`; router tests in `src/tests/_internal/server/routers/test_gateways.py`; factories `create_volume` (testing/common.py:1069), `create_gateway` (:639), `create_gateway_compute` (:683). + +## RECIPE: add a new resource type "endpoint" (derived from volumes, the simpler blueprint) + +1. **Core models** — new `src/dstack/_internal/core/models/endpoints.py`: `EndpointStatus(str, Enum)` (submitted/provisioning/active(or running)/failed), `EndpointConfiguration(CoreModel)` with `type: Literal["endpoint"] = "endpoint"`, `name: Optional[str]`, `model: str`, `env`, + inherit/embed ProfileParams (`core/models/profiles.py:310`) the way run configurations do, `EndpointSpec(CoreModel)` {configuration, configuration_path}, `Endpoint(CoreModel)` (API representation: id, name, project_name, user, configuration, created_at, status, status_message, ...), optional `EndpointPlan`. +2. **Register the config type** — `core/models/configurations.py`: add `ENDPOINT = "endpoint"` to `ApplyConfigurationType` (L1384); add `EndpointConfiguration` to `AnyApplyConfiguration` (L1393), `BaseApplyConfiguration.__root__` union (L1411), and `AnyDstackConfiguration` (L1440, feeds the JSON schema). +3. **DB model** — `server/models.py`: `class EndpointModel(PipelineModelMixin, BaseModel)` with UUID pk, name String(100), project FK CASCADE, user FK, `status EnumAsString(EndpointStatus, 100)`, `status_message Text`, `created_at`/`last_processed_at NaiveDateTime`, `to_be_deleted Boolean server_default=false()`, `deleted`/`deleted_at` if soft-delete (recommended, like volumes), `configuration Text` (JSON), any provisioning-data Text column, UniqueConstraint("project_id","name") or the volumes-style deleted==False partial index `ix_endpoints_pipeline_fetch_q` on `last_processed_at.asc()`. +4. **Alembic migration** — new file under `server/migrations/versions/2026/`, following the batch_alter_table/NaiveDateTime/UUIDType pattern. +5. **Service module** — `server/services/endpoints.py` copying services/volumes.py structure: `create_endpoint(session, project, user, configuration, pipeline_hinter)` (plugin policies -> validation raising ServerClientError -> `endpoint_names_{project.name}` lock -> ResourceExistsError on dup -> insert SUBMITTED -> events.emit -> commit -> `pipeline_hinter.hint_fetch(EndpointModel.__name__)`), `delete_endpoints(...)` (lock_ctx + set to_be_deleted), `list_/get_by_name`, `endpoint_model_to_endpoint`, `switch_endpoint_status`, `emit_endpoint_status_change_event`. +6. **Events** — `core/models/events.py` add `EventTargetType.ENDPOINT`; `server/services/events.py` add EndpointModel to `Target.from_model` union and isinstance chain. +7. **Pipeline** — `server/background/pipeline_tasks/endpoints.py` cloning volumes.py: `EndpointPipelineItem(PipelineItem)` {status, to_be_deleted}, `EndpointPipeline(Pipeline[...])` with `hint_fetch_model_name -> EndpointModel.__name__`, `EndpointFetcher.fetch` (SUBMITTED|PROVISIONING|to_be_deleted filter — include PROVISIONING like gateways since endpoint provisioning is long-running/polled), `EndpointWorker.process` dispatching `_process_submitted_endpoint` / `_process_provisioning_endpoint` / `_process_to_be_deleted_endpoint`, `_EndpointUpdateMap(ItemUpdateMap)`, write-back via conditional UPDATE + `set_processed_update_map_fields`/`set_unlock_update_map_fields`. Register in `PipelineManager.__init__` list (background/pipeline_tasks/__init__.py:35-48). NOTE: worker `lock_timeout=30s` with heartbeat renewal means long operations (LLM agent run) are fine while the worker holds the item, since the Heartbeater renews the lease — but the whole operation blocks one of the 10 workers; the gateway pattern (state machine with short steps polled every 15s, child resources doing the long work) fits an agent-driven flow better. +8. **Schemas + router** — `server/schemas/endpoints.py` (Create/Get/List/Delete request models) and `server/routers/endpoints.py` with `project_router = APIRouter(prefix="/api/project/{project_name}/endpoints", tags=["endpoints"], responses=get_base_api_additional_responses())`; use `Depends(ProjectMember())` (volumes precedent) or `Depends(ProjectAdmin())` (gateways precedent) and `Depends(get_pipeline_hinter)` on create; register in `app.py` `create_app()` next to L257-259. +9. **API client** — `dstack/api/server/_endpoints.py` `EndpointsAPIClient(APIClientGroup)` with list/get/create/delete; add property to `APIClient` in `dstack/api/server/__init__.py`. +10. **CLI** — `cli/services/configurators/endpoint.py` `EndpointConfigurator(BaseApplyConfigurator[EndpointConfiguration])` with `TYPE = ApplyConfigurationType.ENDPOINT`, apply_configuration (plan print, create, poll until ACTIVE/FAILED, exit(1) on FAILED) and delete_configuration; add to `apply_configurators_mapping` in `cli/services/configurators/__init__.py:27`. Optionally `cli/commands/endpoint.py` `EndpointCommand` (list/delete) registered in `cli/main.py`. +11. **Plugins (optional)** — add `EndpointSpec` to the `ApplySpec` TypeVar in `dstack/plugins/_models.py:8` if `apply_plugin_policies` should run on endpoint specs (or skip plugin policies initially — but then don't call `apply_plugin_policies` in the service, since it's typed against that TypeVar). +12. **Tests** — factory `create_endpoint` in `server/testing/common.py`; pipeline tests in `src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py`; router tests in `src/tests/_internal/server/routers/`. + +## Gotchas +1) The old `background/tasks/process_*` layout is gone — processing now lives in `background/pipeline_tasks/` (Pipeline/Fetcher/Worker/Heartbeater classes) and `background/scheduled_tasks/` (APScheduler). Do not plan APIs like `process_submitted_volumes()` as a scheduled job; new resources get a Pipeline subclass registered in `PipelineManager.__init__`. +2) Status must NOT be written by assigning `model.status` in API handlers except via `switch_volume_status`/`switch_gateway_status` (VolumeModel.status docstring says so); pipelines never mutate the ORM object for final state — they write through `update(...).where(id==..., lock_token==...)` with an ItemUpdateMap so a stolen lock aborts the write. A new pipeline must copy this pattern exactly (including `set_unlock_update_map_fields` + `set_processed_update_map_fields` + `resolve_now_placeholders`). +3) Deletion is asynchronous and marker-based: API sets `to_be_deleted=True` only after acquiring row locks with a 10-retry loop and raises ServerClientError if rows are currently locked by a pipeline (`lock_expires_at IS NOT NULL`). Volumes soft-delete (`deleted`, `deleted_at`); gateways HARD-delete the row (`delete(GatewayModel)`) — gateways have no `deleted` column. Pick one consciously; the volumes fetch query and index both filter `deleted == False`. +4) `pipeline_hinter.hint_fetch()` takes the model CLASS name string (`VolumeModel.__name__`), not the table name, and only hints pipelines on the same server replica; correctness relies on the fetcher's polling (fetch_delays 0.5-5s + min_processing_interval), the hint is just latency optimization. Routers get it via `Depends(get_pipeline_hinter)`; background code can use `get_pipeline_manager().hinter`. +5) The fetch query has subtle conditions to copy verbatim: `or_(last_processed_at <= now - min_processing_interval, last_processed_at == created_at)` (process new items immediately), `lock_owner IS NULL OR lock_owner == .__name__` (lets other components take exclusive ownership), and `with_for_update(skip_locked=True, key_share=True, of=Model)`. Also `created_at` must equal `last_processed_at` on insert for the immediate-processing trick to work (services set both to the same `now`). +6) Worker `lock_timeout` is 30s but the Heartbeater renews leases every ~15s while an item is being processed, so long-running work in a worker is safe; however an unhandled exception leaves the row locked until expiry (up to 30s delay before retry) and the item is simply re-fetched — process functions are expected to be idempotent/retryable. +7) `events.emit()` does NOT commit; it must be called inside the transaction that commits the change. `Target.from_model` raises ValueError for unregistered model types — adding a resource without touching events.py + EventTargetType will crash the first emit. +8) Permission choice differs between the two blueprints: volumes use `ProjectMember()` for create/delete; gateways use `ProjectAdmin()` for everything mutating. Both return `Tuple[UserModel, ProjectModel]`. Names like `ProjectUser` or `ProjectRead` do not exist. +9) Name-uniqueness on create uses a three-part dance: sqlite -> `await session.commit()` first; postgres -> `pg_advisory_xact_lock(string_to_lock_id(ns))`; plus the in-process lockset. Volumes additionally rely on this instead of a DB unique constraint (VolumeModel has NO unique constraint on (project_id, name) because of soft-deleted rows; GatewayModel HAS `uq_gateways_project_id_name`). +10) `parse_apply_configuration` does two-pass parsing: extras-tolerant `BaseApplyConfiguration.__response__` first, then strict re-parse; volume configs re-dispatch on the `backend` discriminator. A single flat EndpointConfiguration is a "final configuration" and needs no second pass, but it must be added to BOTH `AnyApplyConfiguration` and `BaseApplyConfiguration.__root__`. +11) The high-level `dstack.api._public.Client` deliberately exposes only runs/repos/backends; CLI configurators use `self.api.client.` (raw APIClient). Don't plan a `Client.endpoints` collection as "the pattern" — the pattern is an APIClientGroup on the low-level client. +12) `apply_plugin_policies` is typed against `ApplySpec = TypeVar(..., RunSpec, FleetSpec, VolumeSpec, GatewaySpec)` (dstack/plugins/_models.py:8) — calling it with a new spec type without extending the TypeVar breaks the plugin contract (and type checking). +13) Volume plan is computed client-side in the CLI (no /get_plan endpoint for volumes); gateways have server-side get_plan/apply_plan with optimistic-concurrency `current_resource` checking and in-place-update whitelisting (`_can_update_gateway_in_place`). For a new resource, the gateway plan/apply flow is the more modern pattern. +14) `SERVER_BACKGROUND_PROCESSING_ENABLED` can be off (app.py:178) — routers must tolerate a missing pipeline_manager (`get_pipeline_hinter` returns a no-op hinter), i.e., never assume processing happens in-request. +15) The two-level gateway design (GatewayModel + GatewayComputeModel children, each with its own pipeline; parent polls child statuses in its PROVISIONING handler) is the precedent for an endpoint that drives a long provisioning process (e.g., a service run created by an agent): the parent state machine stays cheap and re-entrant, re-processed at most every `min_processing_interval` (15s). diff --git a/endpoint-implementation-research/run-submission-internals.md b/endpoint-implementation-research/run-submission-internals.md new file mode 100644 index 0000000000..12e4233e0c --- /dev/null +++ b/endpoint-implementation-research/run-submission-internals.md @@ -0,0 +1,126 @@ +# run-submission-internals + +## Summary +Server-side run creation is fully feasible from a background task: `dstack._internal.server.services.runs` (now a package at src/dstack/_internal/server/services/runs/) exposes `get_plan`, `apply_plan`, `submit_run`, `stop_runs`, `delete_runs`, and `get_run_by_name/-id` that take only (AsyncSession, UserModel, ProjectModel, RunSpec/plan) — no HTTP request context. Repo-less runs are first-class: leaving `run_spec.repo_id`/`repo_data` as None makes the server default to the virtual repo (id "none") and auto-create the RepoModel row; no code upload is needed. There is NO existing precedent of the server creating whole runs from background tasks (submit_run/apply_plan are only called from routers/runs.py), but the server does create replacement/scale-up JobModels in the RunPipeline (rolling deployments, retries, scheduled runs), and the run/job/instance state machines are driven entirely by the multi-replica-safe pipeline_tasks framework (row locks via lock_owner/lock_token/lock_expires_at + FOR UPDATE SKIP LOCKED). There is no dedicated system user; the closest is the startup-created "admin" user (`get_or_create_admin_user`), and UserModel stores an encrypted API token plus per-user SSH keypair (required for run submission). RunModel has no tags/labels column — linking an endpoint to its run must be done via run_name convention or an FK on the new endpoint table. + +## Key files +- src/dstack/_internal/server/services/runs/__init__.py — get_plan, apply_plan, submit_run, stop_runs, delete_runs, get_run, get_run_by_name, get_run_by_id, get_run_model_by_name, run_model_to_run, switch_run_status, create_job_model_for_new_submission, is_job_ready, _generate_run_name, _get_run_repo_or_error — The whole server-side run lifecycle API. All functions take (session, user/project models, RunSpec) — callable from background tasks. +- src/dstack/_internal/server/services/runs/spec.py — validate_run_spec_and_set_defaults, check_can_update_run_spec, can_update_run_spec — Sets virtual-repo defaults (repo_id='none', VirtualRunRepoData) and requires ssh_key_pub or user.ssh_public_key. +- src/dstack/_internal/core/models/runs.py — RunSpec (l.522), RunStatus (l.652), RunTerminationReason (l.91), JobStatus (l.62), JobTerminationReason (l.134), Run (l.675), RunPlan (l.715), ApplyRunPlanInput (l.730), ServiceSpec (l.643) — Core pydantic models; RunStatus.finished_statuses() = [TERMINATED, FAILED, DONE]. +- src/dstack/_internal/core/models/repos/virtual.py — DEFAULT_VIRTUAL_REPO_ID='none', VirtualRepoInfo, VirtualRunRepoData, VirtualRepo — Virtual (repo-less) repo mechanism. +- src/dstack/_internal/server/routers/runs.py — root_router /api/runs/list; project_router /api/project/{project_name}/runs/{get,get_plan,apply,stop,delete,submit(deprecated)} — The only place submit_run/apply_plan/stop_runs are invoked today; shows required call pattern incl. pipeline_hinter and ssh-key refresh. +- src/dstack/_internal/server/schemas/runs.py — GetRunPlanRequest, ApplyRunPlanRequest, StopRunsRequest, SubmitRunRequest — API request schemas. +- src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py — RunPipeline, RunFetcher, RunWorker — Multi-replica-safe processing pattern (lock_token/lock_expires_at/lock_owner + FOR UPDATE SKIP LOCKED) to copy for an EndpointPipeline. +- src/dstack/_internal/server/background/pipeline_tasks/runs/active.py — process_active_run, _get_active_run_transition, _analyze_active_run — Run status semantics: RUNNING if any replica job RUNNING; FAILED via TERMINATING+termination_reason; rolling deployment/scaling create JobModels here (server-side job creation precedent). +- src/dstack/_internal/server/background/pipeline_tasks/__init__.py — PipelineManager, PipelineHinter, start_pipeline_tasks, get_pipeline_manager — Where a new pipeline would be registered; started from app.py lifespan when SERVER_BACKGROUND_PROCESSING_ENABLED. +- src/dstack/_internal/server/services/services/__init__.py — register_service, _register_service_in_server, _register_service_in_gateway — Called synchronously inside submit_run for service runs; produces ServiceSpec url (/proxy/services/// or gateway URL) and model mapping. +- src/dstack/_internal/server/services/users.py — get_or_create_admin_user (l.45), create_user (l.160), get_user_model_by_name (l.327), refresh_ssh_key (l.232) — Identity options for a background task; users get RSA ssh keypair on creation; token stored encrypted. +- src/dstack/_internal/server/services/projects.py — get_project_model_by_name (l.572), get_project_model_by_name_or_error (l.589) — How to load ProjectModel (with backends+members joined) to act 'as' a project. +- src/dstack/_internal/server/models.py — RunModel (l.405), UserModel (l.210), PipelineModelMixin (l.204) — RunModel columns — no tags/labels column; UserModel.token is EncryptedString + token_hash. +- src/dstack/_internal/server/services/repos.py — create_or_update_repo (l.101), get_repo_model (l.317), get_code_model (l.331) — Virtual repo row auto-created by _get_run_repo_or_error during submit_run. +- src/dstack/api/_public/runs.py — RunCollection.get_run_plan (l.470), apply_plan (l.547), apply_configuration (l.585) — Client-side reference for how RunSpec is constructed for repo-less configs. +- src/dstack/_internal/cli/services/configurators/run.py — BaseRunConfigurator.apply_configuration (l.87), ServiceConfigurator (l.716) — CLI apply flow for services; falls back to init_default_virtual_repo when no repo. + +## Details +# Ground truth: server-side run (service) creation, monitoring, termination + +## 1. services/runs package (src/dstack/_internal/server/services/runs/) +Package files: `__init__.py` (1243 l.), `plan.py`, `replicas.py`, `spec.py`, `router_worker_sync.py`, `service_router_worker_sync.py`. + +Exact signatures (all in `src/dstack/_internal/server/services/runs/__init__.py`): + +- `async def get_plan(session: AsyncSession, project: ProjectModel, user: UserModel, run_spec: RunSpec, max_offers: Optional[int], legacy_repo_dir: bool = False) -> RunPlan` (l.528). Applies plugin policies, validates spec, computes `job_plans` via `get_job_plans` (runs/plan.py), detects `action` CREATE vs UPDATE. Optional step — NOT required before submit. +- `async def apply_plan(session: AsyncSession, user: UserModel, project: ProjectModel, plan: ApplyRunPlanInput, force: bool, pipeline_hinter: Optional[PipelineHinterProtocol] = None, legacy_repo_dir: bool = False) -> Run` (l.587). Applies plugin policies; if `run_spec.run_name is None` or no active run with that name → delegates to `submit_run`; else attempts in-place update (only fields in `_UPDATABLE_SPEC_FIELDS`/`_CONF_UPDATABLE_FIELDS`, spec.py:26-63), bumping `deployment_num` (rolling deployment for services). +- `async def submit_run(session: AsyncSession, user: UserModel, project: ProjectModel, run_spec: RunSpec, pipeline_hinter: Optional[PipelineHinterProtocol] = None) -> Run` (l.681). Flow: `validate_run_spec_and_set_defaults(user, run_spec)` → `_get_run_repo_or_error` (auto-creates virtual repo) → `get_project_secrets_mapping` → run-name lock (`pg_advisory_xact_lock` / lockset namespace `run_names_{project.name}`) → generate name if None (`_generate_run_name`, l.1134, adjective-animal-N) else `delete_runs` of finished same-name run → `_validate_run` (volumes) → creates `RunModel(...)` (l.734: initial_status=SUBMITTED, or PENDING+0 replicas if `merged_profile.schedule`; `desired_replica_count=1` — real value set by RunPipeline; `priority=run_spec.configuration.priority`; `deployment_num=0`) → for services: `await services.register_service(session, run_model, run_spec)` then per replica-group creates jobs via `get_jobs_from_run_spec(run_spec=..., secrets=..., replica_num=..., replica_group_name=...)` and `create_job_model_for_new_submission(run_model, job, status=JobStatus.SUBMITTED)` (l.833), plus `ensure_service_router_worker_sync_row(session, run_model, run_spec)` → `session.commit()` → `pipeline_hinter.hint_fetch("JobModel"/"RunModel")` if provided → returns `await get_run_by_id(...)`. +- `async def stop_runs(session: AsyncSession, user: UserModel, project: ProjectModel, runs_names: List[str], abort: bool, pipeline_hinter: Optional[PipelineHinterProtocol] = None)` (l.865). Sets `termination_reason = RunTerminationReason.ABORTED_BY_USER|STOPPED_BY_USER`, `switch_run_status(session, run_model, RunStatus.TERMINATING, actor=UserActor)`, `skip_min_processing_interval=True`; the RunPipeline finishes termination. Commits. +- `async def delete_runs(session, user, project, runs_names: List[str])` (l.909) — only finished runs; sets `deleted=True`. +- Reads: `async def get_run(session, project, run_name=None, run_id=None) -> Optional[Run]` (l.456); `get_run_model_by_name(session, project, run_name) -> Optional[RunModel]` (l.477); `get_run_by_name` (l.496); `get_run_by_id` (l.507); `def run_model_to_run(run_model, include_jobs=True, job_submissions_limit=None, return_in_api=False, include_sensitive=False, include_job_connection_info=False, loaded_jobs=None) -> Run` (l.949) — populates `run.service` from `run_model.service_spec` JSON. +- `def switch_run_status(session, run_model, new_status: RunStatus, actor: events.AnyActor = events.SystemActor())` (l.97) — the ONLY sanctioned way to change RunModel.status (emits event). +- `def get_run_spec(run_model) -> RunSpec` (l.154) — parses `run_model.run_spec` JSON. +- `def is_job_ready(probes: Iterable[ProbeModel], probe_specs: Iterable[ProbeSpec]) -> bool` (l.1226). + +spec.py: `def validate_run_spec_and_set_defaults(user: UserModel, run_spec: RunSpec, legacy_repo_dir: bool = False)` (l.66): validates run_name against `^[a-z][a-z0-9-]{1,40}$`; sets `repo_id=DEFAULT_VIRTUAL_REPO_ID`/`repo_data=VirtualRunRepoData()` when both None (l.88-91); sets priority default; requires ssh key (l.128-132: `raise ServerClientError("ssh_key_pub must be set if the user has no ssh_public_key")`). + +## 2. Repo requirement / virtual repo +`src/dstack/_internal/core/models/repos/virtual.py`: `DEFAULT_VIRTUAL_REPO_ID = "none"` (l.11), `class VirtualRepoInfo(BaseRepoInfo)` with `repo_type: Literal["virtual"]`, `class VirtualRunRepoData(VirtualRepoInfo)`, `class VirtualRepo(Repo)` (programmatic files via `add_file`, tarred by `write_code_file`). + +Server mechanism: a run CAN be submitted with `run_spec.repo_id=None, repo_data=None`; `validate_run_spec_and_set_defaults` fills the virtual defaults, then `_get_run_repo_or_error` (runs/__init__.py:1185) sees `repo_data.repo_type == "virtual"` and calls `repos_services.create_or_update_repo(session, project, repo_id, repo_info)` (repos.py:101) which upserts the RepoModel row — i.e. NO prior `/repos/init` call is needed server-side. If `repo_code_hash` is None, no code blob is fetched for the runner (`_get_job_code` in background/pipeline_tasks/jobs_running.py:1745 returns None when `code_hash is None`). + +CLI behavior for repo-less configs: `BaseRunConfigurator.apply_configuration` (cli/services/configurators/run.py:112-114) — if `self.get_repo(...)` returns None it calls `init_default_virtual_repo(api)` (cli/services/repos.py:34-37: `repo = VirtualRepo(); api.repos.init(repo)`). The SDK (`RunCollection.get_run_plan`, api/_public/runs.py:498-542) builds `RunSpec(run_name=configuration.name, repo_id=repo.repo_id, repo_data=repo.run_repo_data, repo_code_hash=None-if-no-files, file_archives=..., configuration=..., profile=..., ssh_key_pub=None → server-managed user key)`. + +## 3. dstack apply flow for a service +CLI: `ServiceConfigurator` (cli/services/configurators/run.py:716) extends `BaseRunConfigurator.apply_configuration` (l.87) → `self.api.runs.get_run_plan(configuration, repo, configuration_path, profile, ssh_identity_file)` (api/_public/runs.py:470) → HTTP `POST /api/project/{project_name}/runs/get_plan` → router `get_plan` (server/routers/runs.py:112-140; body `GetRunPlanRequest{run_spec: RunSpec, max_offers: Optional[int]}`; deps `ProjectMember()`, `get_client_version`, `use_legacy_repo_dir`) → `runs.get_plan(...)`. Then `self.api.runs.apply_plan(run_plan, repo, reserve_ports)` (api/_public/runs.py:547; uploads code tar via `repos.upload_code` only if `repo.has_code_to_write()`) → `POST /api/project/{project_name}/runs/apply` (routers/runs.py:143-171; body `ApplyRunPlanRequest{plan: ApplyRunPlanInput{run_spec, current_resource}, force: bool}`) → `runs.apply_plan(...)`. Router also calls `users.refresh_ssh_key(session, actor=user)` when user has no ssh key. Other endpoints: `POST /api/runs/list` (root router), `POST /api/project/{p}/runs/get`, `/stop` (`StopRunsRequest{runs_names, abort}`), `/delete`, `/submit` (deprecated, body `SubmitRunRequest{run_spec}`). + +## 4. Stop/terminate + status semantics +Terminate programmatically: `runs.stop_runs(session, user, project, runs_names, abort, pipeline_hinter=None)` — this is the idiomatic way (routers use it); it flips status to TERMINATING and RunPipeline (`_process_terminating_item` → `process_terminating_run` in background/pipeline_tasks/runs/terminating.py:61) unregisters the service from the gateway (`_unregister_service`, terminating.py:146) and terminates jobs. + +`RunStatus` (core/models/runs.py:652): PENDING, SUBMITTED, PROVISIONING, RUNNING, TERMINATING, TERMINATED, FAILED, DONE; `finished_statuses() = [TERMINATED, FAILED, DONE]`; `is_finished()`. `RunTerminationReason` (l.91): ALL_JOBS_DONE→DONE, JOB_FAILED→FAILED, RETRY_LIMIT_EXCEEDED→FAILED, STOPPED_BY_USER→TERMINATED, ABORTED_BY_USER→TERMINATED, SERVER_ERROR→FAILED (`to_status()`, l.114). + +Service semantics (background/pipeline_tasks/runs/active.py `_get_active_run_transition`, l.369): run is RUNNING when ANY replica contributes RUNNING (job RUNNING); PROVISIONING when best is PROVISIONING/PULLING; FAILED (via TERMINATING + JOB_FAILED/RETRY_LIMIT_EXCEEDED) when a replica failed non-retryably; PENDING when all replicas are retrying. `Run.error` = `termination_reason.to_error()`; `Run.status_message` may show "pulling"/"retrying" (runs/__init__.py:1079). + +"Ready" for services is stronger than RUNNING: a replica is registered on the gateway/in-server proxy only when its probes are all ready — `_maybe_register_replica` (jobs_running.py:1119-1130) gates on `is_job_ready(job_model.probes, job_spec.probes)`; `is_probe_ready(probe, spec) = probe.success_streak >= spec.ready_after` (services/probes.py:9). If `ServiceConfiguration.model` is set and `probes` omitted, a default `/v1/chat/completions` probe is applied (configurations.py:1019-1026). Service URL: `Run.service: Optional[ServiceSpec]` — for in-server proxy `url = /proxy/services/{project}/{run_name}/`, model at `/proxy/models/{project}/` (services/services/__init__.py:240-282, `_register_service_in_server`); ServiceSpec.model = `ServiceModelSpec(name, base_url, type)`. + +## 5. Precedent for server-created runs +`grep submit_run(` over src: only routers/runs.py:217 and internal recursion in apply_plan (runs/__init__.py:608,621). `RunModel(` is instantiated exactly once in prod code (runs/__init__.py:734). So: NO existing code path where the server creates a run without a user API call. Closest precedents (server creates JOBS, not runs): (a) retry — `_build_retry_job_models` (active.py:455); (b) service auto-scaling — `_build_service_scaling_maps` (active.py:576) / `build_scale_up_job_models` (pipeline_tasks/runs/common.py:56); (c) rolling deployment — `_build_rolling_deployment_maps` (active.py:645); (d) scheduled runs — run stays PENDING and RunFetcher picks it up when `next_triggered_at < now` (pipeline_tasks/runs/__init__.py:157-163), then `process_pending_run` (pending.py:43) creates job models. All of these keep the user-created RunModel and attribute job creation to `events.SystemActor()` (see comment at runs/__init__.py:814-817). + +## 6. Identity for a background task +- No system/global service user exists. `get_or_create_admin_user(session) -> Tuple[UserModel, bool]` (services/users.py:45; username "admin", GlobalRole.ADMIN, token from `DSTACK_SERVER_ADMIN_TOKEN` or uuid4) is created at startup (app.py:142) and is the only guaranteed user. +- `UserModel.token: Mapped[DecryptedString] = mapped_column(EncryptedString(200), unique=True)` + `token_hash` (models.py:218-220); plaintext via `admin.token.get_plaintext_or_error()` (used in app.py:166,198). ssh_private_key/ssh_public_key columns nullable (pre-0.19.33 users); `create_user` (users.py:160) always generates an RSA pair. +- Acting "as" a project: load `ProjectModel` via `projects.get_project_model_by_name_or_error(session, project_name)` (services/projects.py:589; joinedloads backends+members) and pass it straight to the runs service functions — the service layer does no auth (auth is only `ProjectMember()` in security/permissions.py:112, a FastAPI dep resolving Bearer token → (UserModel, ProjectModel)). Idiomatic choice for an endpoint feature: store the endpoint creator's `user_id` on the endpoint model and submit runs as that user (events/attribution stay correct); events emitted by the background machinery itself use `events.SystemActor()` (services/events.py:34; `AnyActor = Union[SystemActor, UserActor]`, `emit(session, message, actor, targets)` l.171). +- Multi-replica-safe background patterns: (a) Pipeline framework — subclass `Pipeline[ItemT]`/`Fetcher`/`Worker`/`Heartbeater` (background/pipeline_tasks/base.py; items carry lock_token/lock_expires_at; fetch uses `.with_for_update(skip_locked=True, key_share=True)` + `lock_owner` — see RunFetcher.fetch, pipeline_tasks/runs/__init__.py:132-233), register in `PipelineManager.__init__` (pipeline_tasks/__init__.py:35-49), model gets `PipelineModelMixin` columns (models.py:204-207); or (b) apscheduler scheduled task — add in `start_scheduled_tasks()` (background/scheduled_tasks/__init__.py:37). Both started in app.py lifespan iff `settings.SERVER_BACKGROUND_PROCESSING_ENABLED` (app.py:178-183). DB sessions in background code come from `get_session_ctx()` (server/db.py). + +## 7. Naming and tagging +- Name rule: `validate_dstack_resource_name` → regex `^[a-z][a-z0-9-]{1,40}$` (core/services/__init__.py:6-12). Auto-generation: `generate_name()` (utils/random_names.py:253, adjective-animal) + `-{idx}` uniqueness loop scoped to project + non-deleted (`_generate_run_name`, runs/__init__.py:1134). +- Uniqueness: enforced among non-deleted runs per project under an advisory lock; submitting with an existing name of a FINISHED run marks the old run deleted (submit_run l.716-718); an ACTIVE same-name run makes apply_plan try in-place update or raise "Cannot override active run". +- Tags/labels: RunModel (models.py:405-465) has NO tag column. `ProfileParams.tags: Optional[Dict[str,str]]` (core/models/profiles.py:462) rides inside run_spec JSON and is propagated to backend cloud resources — not SQL-filterable. To link runs to an owning endpoint: use a deterministic run_name (e.g. derived from endpoint name) and/or an FK from the new EndpointModel to `RunModel.id`; RunModel.fleet_id (models.py:422) is prior art for run→resource attachment. + +## Minimal server-side call sequence (verified building blocks) +```python +from dstack._internal.server.db import get_session_ctx +from dstack._internal.server.services import projects as projects_services +from dstack._internal.server.services import runs as runs_services +from dstack._internal.server.services import users as users_services +from dstack._internal.core.models.runs import RunSpec, RunStatus +from dstack._internal.core.models.configurations import ServiceConfiguration + +async with get_session_ctx() as session: + project = await projects_services.get_project_model_by_name_or_error(session, project_name) + user = await users_services.get_user_model_by_name(session, username) # e.g. endpoint creator or "admin" + run_spec = RunSpec( + run_name="my-endpoint-svc", # or None -> auto-generated + configuration=ServiceConfiguration(commands=[...], port=8000, model="", env=..., replicas=...), + # repo_id / repo_data left None -> virtual repo defaults applied server-side + ssh_key_pub=None, # ok iff user.ssh_public_key is set + ) + run = await runs_services.submit_run(session=session, user=user, project=project, + run_spec=run_spec, pipeline_hinter=None) # commits + +# monitor (poll): +async with get_session_ctx() as session: + project = await projects_services.get_project_model_by_name_or_error(session, project_name) + run = await runs_services.get_run_by_name(session=session, project=project, run_name=name) + # run.status (RunStatus), run.error, run.service.url / run.service.model.base_url + # readiness: run.status == RunStatus.RUNNING (+ probe-based gateway registration already gated server-side) + +# terminate: +async with get_session_ctx() as session: + await runs_services.stop_runs(session=session, user=user, project=project, + runs_names=[name], abort=False, pipeline_hinter=None) # commits +``` +Testing helpers exist: `create_user`, `create_project`, `create_repo`, `create_run(session, project, repo, user, run_name=None, status=RunStatus.SUBMITTED, run_spec=None, ...) -> RunModel` in src/dstack/_internal/server/testing/common.py (create_run at l.365). + +Not verified/does not exist: no `runs/apply.py` module (apply logic lives in runs/__init__.py); no server-side "preset service config" store; no endpoint-like resource today; no system user; no run tag columns. + +## Gotchas +1) `submit_run` COMMITS the session multiple times (also via its internal `delete_runs` call) and uses cross-process locking (`pg_advisory_xact_lock` on Postgres / in-memory lockset on SQLite) for run-name uniqueness — call it with a fresh session from `get_session_ctx()` in a background task, never inside another transaction you expect to control. +2) `validate_run_spec_and_set_defaults` (called by both apply_plan and submit_run) raises ServerClientError unless `run_spec.ssh_key_pub` is set OR `user.ssh_public_key` is non-empty (spec.py:128-132). Users created via `create_user` always have keys; users created via testing helpers or pre-0.19.33 may not. +3) `submit_run` does NOT apply plugin policies — only `get_plan`/`apply_plan` call `apply_plugin_policies`. If the endpoint feature should respect plugins, go through `apply_plan(plan=ApplyRunPlanInput(run_spec=...), force=True)` rather than `submit_run` directly. Note `apply_plan` with `run_spec.run_name=None` just delegates to submit_run. +4) For services, `submit_run` synchronously calls `services.register_service` (runs/__init__.py:758-760, marked FIXME) — it can raise ResourceNotExistsError (gateway referenced but missing, or `FORBID_SERVICES_WITHOUT_GATEWAY` set) or ServerClientError (autoscaling/rate_limits without gateway). An endpoint background task must handle these as submission failures. +5) No membership/authorization checks exist in the service layer — `ProjectMember()` in routers is the only gate. A background task passing any UserModel works; the run is attributed to that user (RunModel.user_id, events UserActor). +6) There is NO system user. Only the `admin` user (username "admin", GlobalRole.ADMIN) is guaranteed to exist (created in app.py:142 at startup). If endpoints should be attributed to the submitting user, store the endpoint's creator user_id and pass that UserModel. +7) RunModel has NO tags column. `ProfileParams.tags` (profiles.py:462) exists in run_spec JSON only (propagated to cloud resources, not SQL-queryable). Link endpoint→run via run_name (regex `^[a-z][a-z0-9-]{1,40}$`, unique among non-deleted runs per project) or an FK to RunModel.id on the new endpoint table. +8) Run status is not a reliable "ready" signal for services: RunStatus.RUNNING means ≥1 replica job is RUNNING, but the replica is only registered on the gateway/proxy after all its probes are ready (`is_job_ready`, jobs_running.py:1119-1130; `is_probe_ready` = success_streak >= spec.ready_after). If `model:` is set and probes omitted, a default /v1/chat/completions probe is added. For endpoint health-checking, either rely on probes+RUNNING or poll the service URL directly. +9) Do NOT set RunModel.status directly — use `switch_run_status()` (models.py:437 docstring) so an event is emitted; run termination is done by setting status TERMINATING + termination_reason (+ `skip_min_processing_interval=True`) and letting RunPipeline finish it — `stop_runs` does exactly this. +10) Background processing can be disabled per-replica (`SERVER_BACKGROUND_PROCESSING_ENABLED`, app.py:178); pipelines poll the DB, and the optional `pipeline_hinter` (services/pipelines.py, a FastAPI dependency) only reduces latency — passing None from a background task is fine, or use `get_pipeline_manager().hinter` in-process. +11) `submit_run` deletes any existing finished run with the same run_name (runs/__init__.py:716) — reusing a deterministic endpoint-derived run name will silently erase previous run history. +12) Runs with `schedule` in the profile start as PENDING with 0 replicas (runs/__init__.py:730-732); otherwise initial status SUBMITTED. diff --git a/endpoint-implementation-research/service-runtime-health.md b/endpoint-implementation-research/service-runtime-health.md new file mode 100644 index 0000000000..76c69df5db --- /dev/null +++ b/endpoint-implementation-research/service-runtime-health.md @@ -0,0 +1,144 @@ +# service-runtime-health + +## Summary +Services with `model:` set get an OpenAI-compatible endpoint via a ServiceSpec stored on RunModel; without a gateway, the in-server proxy serves `/proxy/services///...` and `/proxy/models//...` (the latter marked deprecated but functional). Auth is a Bearer token of any project member, but server code never needs HTTP: probes already health-check model endpoints in-process by SSH-tunneling to the replica's service port (`get_service_replica_client(job)` in `job_replica_http_client.py`) and POSTing `{prefix}/chat/completions`. A default chat-completions probe is auto-generated for OpenAI-format models, executed every 3s by the multi-replica-safe `process_probes` scheduled task, and probe success (`success_streak >= ready_after`) gates the `JobModel.registered` flag which is what "replica ready" means. The endpoint health-checker should reuse the exact probe mechanism (job-level SSH tunnel + httpx over UDS), which works identically for gateway and non-gateway services. + +## Key files +- src/dstack/_internal/server/background/scheduled_tasks/probes.py — process_probes, _process_probe_async, _execute_probe — THE reference implementation for health-checking a deployed model: batch-locks due ProbeModels (multi-replica safe), tunnels to the replica, POSTs the probe request, updates success_streak. `_execute_probe` (lines 106-126) is exactly what the endpoint health-checker should copy. +- src/dstack/_internal/server/services/jobs/job_replica_http_client.py — get_service_replica_client(job: JobModel) -> AsyncGenerator[httpx.AsyncClient, None] — Async context manager giving an httpx client to a job's service port over an SSH tunnel + Unix socket. No HTTP auth, no gateway needed. Lines 21-27. +- src/dstack/_internal/server/services/jobs/job_replica_tunnel.py — get_service_replica_tunnel, SSH_CONNECT_TIMEOUT — Builds the tunnel via container_ssh_tunnel(job, forwarded_sockets=[SocketPair(remote=IPSocket('localhost', job_spec.service_port), local=UnixSocket(...))]). +- src/dstack/_internal/server/services/jobs/configurators/base.py — _probes (line 410), _probe_config_to_spec (458), _openai_model_probe_spec (472) — Default probe generation: if ServiceConfiguration.probes is None and model is OpenAIChatModel, generates POST {prefix}/chat/completions probe with body {model, messages:[{role:user,content:'hi'}], max_tokens:1}, timeout 30s. +- src/dstack/_internal/server/services/services/__init__.py — register_service (49), _register_service_in_gateway (99), _register_service_in_server (240), _get_service_spec (320) — Where `model:` becomes URLs. In-server: service_url=/proxy/services/{project}/{run}/ (273), model_url=service_url+prefix for openai / /proxy/models/{project}/ for tgi (274-277). Gateway: https://{run_name}.{wildcard_domain}, model at gateway.{wildcard_domain} (160-172). Result stored as run_model.service_spec JSON (96). +- src/dstack/_internal/server/services/proxy/repo.py — ServerProxyRepo.get_service (47), list_models (143), get_model (172) — In-server proxy data source. get_service only returns runs with gateway_id IS NULL, JobStatus.RUNNING, registered==True, job_num==0. list_models requires RunStatus.RUNNING + service_spec.options['openai']['model']. +- src/dstack/_internal/proxy/lib/services/service_connection.py — ServiceConnectionPool, ServiceConnection, get_service_replica_client(service, repo, service_conn_pool) (140) — Proxy-lib variant of replica client (pooled, persistent tunnels). Used by the in-server proxy and model proxy routers. +- src/dstack/_internal/proxy/lib/services/model_proxy/clients/openai.py — OpenAIChatCompletions.generate/stream — Existing typed client that POSTs {prefix}/chat/completions given an httpx.AsyncClient — reusable for a higher-level health check with response validation. +- src/dstack/_internal/proxy/lib/routers/model_proxy.py — get_models, post_chat_completions — OpenAI-compatible HTTP API: GET /proxy/models/{project}/models, POST /proxy/models/{project}/chat/completions. Mounted in app.py:261 with deprecated=True (still functional). +- src/dstack/_internal/server/services/probes.py — probe_model_to_probe, is_probe_ready — is_probe_ready(probe, spec) = probe.success_streak >= spec.ready_after. +- src/dstack/_internal/server/services/runs/__init__.py — is_job_ready (1226), submit_run (register_service call at 760), run_model_to_run (service_spec at 979-1004) — is_job_ready = all probes ready; Run.service: Optional[ServiceSpec] populated from RunModel.service_spec. +- src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py — _initialize_running_job_probes (1101), _maybe_register_replica (1119), _register_service_replica (1162) — Probes created when job reaches RUNNING; JobModel.registered set True only when all probes ready (and gateway registration succeeds if gateway-based). +- src/dstack/_internal/core/models/configurations.py — ProbeConfig (365), ServiceConfiguration.model (993), .probes (1019), convert_model validator (1058), probe constants (61-71) — model: str is coerced to OpenAIChatModel(format='openai'); OPENAI_MODEL_PROBE_TIMEOUT=30, DEFAULT_PROBE_INTERVAL=15, DEFAULT_PROBE_READY_AFTER=1. +- src/dstack/_internal/core/models/runs.py — ProbeSpec (245), JobSpec.probes/service_port (299-301), Probe (390), JobSubmission.probes (428), ServiceModelSpec (635), ServiceSpec (643), RunStatus (652) — ProbeSpec is the resolved probe stored in JobSpec; Probe(success_streak) surfaces per-submission probe state to clients. +- src/dstack/_internal/proxy/lib/deps.py — ProxyAuth, ProxyAuthContext — Bearer-token auth for proxy routes; enforced iff service.auth (service proxy) and always for model proxy. +- src/dstack/_internal/proxy/lib/schemas/model_proxy.py — ChatCompletionsRequest, ChatCompletionsResponse, ChatCompletionsChunk, ModelsResponse — Typed OpenAI-compatible request/response schemas already in the codebase. + +## Details +## 1. The `model:` property on services + +`ServiceConfiguration.model: Optional[AnyModel]` — `src/dstack/_internal/core/models/configurations.py:993-1003`. A plain string is coerced by the `convert_model` validator (configurations.py:1058-1062): + +```python +@validator("model", pre=True) +def convert_model(cls, v: Optional[Union[AnyModel, str]]) -> Optional[AnyModel]: + if isinstance(v, str): + return OpenAIChatModel(type="chat", name=v, format="openai") + return v +``` + +Model types (`src/dstack/_internal/core/models/services.py`): `BaseChatModel` (13), `TGIChatModel` (21), `OpenAIChatModel` (58, `prefix` default `"/v1"` at line 72), `ChatModel = Annotated[Union[TGIChatModel, OpenAIChatModel], Field(discriminator="format")]` (75), `AnyModel = Union[ChatModel]` (76). + +**Registration flow**: `submit_run` calls `await services.register_service(session, run_model, run_spec)` at `src/dstack/_internal/server/services/runs/__init__.py:760` (comment: "FIXME: Register services asynchronously in the background"). `register_service` (`src/dstack/_internal/server/services/services/__init__.py:49-96`) resolves the gateway (explicit ref / default / `gateway: false`) and either: +- `_register_service_in_gateway` (line 99): registers on every gateway connection via `client.register_service(...)` (`src/dstack/_internal/server/services/gateways/client.py:37-74`; when `"openai" in options` it also calls `register_openai_entrypoint` for `gateway.`, lines 52-54). +- `_register_service_in_server` (line 240): no gateway. Rejects SGLang router, non-auto https, autoscaling (min!=max), rate_limits. Then builds URLs (lines 273-277): + +```python +service_url = f"/proxy/services/{run_model.project.name}/{run_model.run_name}/" +if isinstance(run_spec.configuration.model, OpenAIChatModel): + model_url = service_url.rstrip("/") + run_spec.configuration.model.prefix +else: + model_url = f"/proxy/models/{run_model.project.name}/" +``` + +`_get_service_spec` (line 320-331) produces `ServiceSpec(url=service_url)` with `model=ServiceModelSpec(name, base_url=model_url, type)` and `options=get_service_options(configuration)`. `get_service_options` (`src/dstack/_internal/server/services/services/options.py:48-53`) sets `options["openai"] = {"model": conf.model.dict()}` after `complete_service_model` (options.py:10-23) fills TGI `chat_template`/`eos_token` from HuggingFace (network call, may raise `ServerClientError`). Result stored as `run_model.service_spec = service_spec.json()` (services/__init__.py:96). Gateway registration failure raises (`GatewayError` / `ServerClientError`) synchronously from submit. + +`ServiceModelSpec` / `ServiceSpec` — `src/dstack/_internal/core/models/runs.py:635-649`; `Run.service: Optional[ServiceSpec]` (runs.py:693) is populated in `run_model_to_run` (`server/services/runs/__init__.py:979-1004`). + +**In-server model listing**: `ServerProxyRepo.list_models` (`src/dstack/_internal/server/services/proxy/repo.py:143-170`) — runs where `gateway_id IS NULL`, `service_spec IS NOT NULL`, `RunStatus.RUNNING`, and `service_spec.options["openai"]["model"]` present. `get_model` (172-178) picks the newest by `created_at` on name collision. + +## 2. URL formats + +**Without a gateway (in-server proxy, always available unless `settings.FORBID_SERVICES_WITHOUT_GATEWAY`, services/__init__.py:89-95):** +- Service: `/proxy/services/{project_name}/{run_name}/{path}` — router `src/dstack/_internal/server/services/proxy/routers/service_proxy.py:31-49`, mounted at `src/dstack/_internal/server/app.py:260`. +- Model (OpenAI-compatible): `GET /proxy/models/{project_name}/models` and `POST /proxy/models/{project_name}/chat/completions` — router `src/dstack/_internal/proxy/lib/routers/model_proxy.py:26-65`, mounted at app.py:261 **with `deprecated=True`** (still functional; for `format: openai` models the ServiceSpec now points clients at `/proxy/services//` instead, i.e. straight through the service proxy — the model proxy is mainly needed for TGI-format translation). +- These are paths relative to the dstack server's base URL; auth via Bearer token (see below). + +**With a gateway** (services/__init__.py:160-177): service at `{http|https}://{run_name}.{wildcard_domain}`; model endpoint at `{http|https}://gateway.{wildcard_domain}` (or `service_url + model.prefix` for openai format). Requires wildcard domain DNS; served by Nginx on the gateway instance. + +## 3. Auth for calling through the proxy + +- Both proxy routers use `ProxyAuth` (`src/dstack/_internal/proxy/lib/deps.py:87-106`): `HTTPBearer` credentials → `ProxyAuthContext.enforce()` (deps.py:71-84) → `BaseProxyAuthProvider.is_project_member(project_name, token)`. Server implementation: `ServerProxyAuthProvider` (`src/dstack/_internal/server/services/proxy/auth.py:7-12`) → `is_project_member(session, project_name, token)` (`src/dstack/_internal/server/security/permissions.py:278-283`). So the token is **any project member's user token** (the same `Authorization: Bearer ` used for the REST API). +- Model proxy always enforces (`APIRouter(dependencies=[Depends(ProxyAuth(auto_enforce=True))])`, model_proxy.py:23). Service proxy enforces only when `service.auth` is true (`server/services/proxy/services/service_proxy.py:39-40`; `ServiceConfiguration.auth` default `True`, configurations.py:1012). +- **In-process alternative: yes.** Server code does not need HTTP+token at all — see mechanism below. There is no in-process function that "calls the FastAPI route"; instead server code opens the same SSH tunnel the proxy uses. + +## 4. Service probes + +- Config: `ProbeConfig` (`configurations.py:365-459`): `type: Literal["http"]`, `url` (default `/`), `method` (default `get`), `headers`, `body`, `timeout` (default 10s, min 1s), `interval` (default 15s), `ready_after` (default 1), `until_ready` (default False). `ServiceConfiguration.probes: Optional[list[ProbeConfig]]` (configurations.py:1019-1026) — `None` means "default"; docstring: "If `model` is set, defaults to a `/v1/chat/completions` probe." +- Default generation: job configurator `_probes()` (`server/services/jobs/configurators/base.py:410-419`) — explicit probes are converted by `_probe_config_to_spec` (458); otherwise, if `isinstance(model, OpenAIChatModel)`, returns `[_openai_model_probe_spec(model.name, model.prefix)]` (472-491): `POST {prefix}/chat/completions`, JSON body `{"model": , "messages": [{"role": "user", "content": "hi"}], "max_tokens": 1}`, `Content-Type: application/json`, `timeout=OPENAI_MODEL_PROBE_TIMEOUT` (30s, configurations.py:71), `interval=15`, `ready_after=1`. Note: **TGI-format models get no default probe.** Resolved probes live in `JobSpec.probes: list[ProbeSpec]` (`core/models/runs.py:301`, spec class at 245-255). +- DB: `ProbeModel` (`server/models.py:1117-1133`): `id, name, job_id, probe_num, due, success_streak, active`; unique `(job_id, probe_num)`. +- Lifecycle: created when the job first reaches RUNNING — `_initialize_running_job_probes` (`server/background/pipeline_tasks/jobs_running.py:1101-1116`). Evaluated by scheduled task `process_probes` (`server/background/scheduled_tasks/probes.py:29-79`) registered at `IntervalTrigger(seconds=3, jitter=1)` (`server/background/scheduled_tasks/__init__.py:47`). Multi-replica-safe pattern: `get_locker(get_db().dialect_name).get_lockset(ProbeModel.__tablename__)` + `select(...).where(due <= now, active == True).limit(100).with_for_update(skip_locked=True, key_share=True)`; actual HTTP executed off-session via an `AsyncIOScheduler` job (`_process_probe_async`, 82-103) which then updates `success_streak` (reset to 0 on failure, +1 on success) and `due = now + interval`. +- Execution (`_execute_probe`, probes.py:106-126): + +```python +async with get_service_replica_client(probe.job) as client: + resp = await client.request( + method=probe_spec.method, + url="http://dstack" + probe_spec.url, # host is dummy; transport is a UDS + headers=[(h.name, h.value) for h in probe_spec.headers], + content=probe_spec.body, + timeout=probe_spec.timeout, + follow_redirects=False, + ) + return resp.is_success +``` + +- Status reflection: `Probe(success_streak=int)` (`core/models/runs.py:390-391`) in `JobSubmission.probes` (runs.py:428) via `probe_model_to_probe` (`server/services/probes.py:5-6`). Readiness: `is_probe_ready(probe, spec) = probe.success_streak >= spec.ready_after` (`server/services/probes.py:9-10`); `is_job_ready(probes, probe_specs)` = all ready (`server/services/runs/__init__.py:1226-1227`). Probe failures do NOT fail/terminate the job; they only delay/gate readiness. Probes deactivate when the job leaves RUNNING or when `until_ready` and threshold reached (probes.py:63-69). + +## 5. Service readiness today + +- Job statuses: SUBMITTED → PROVISIONING → PULLING → RUNNING → ... (`core/models/runs.py:62-78`); run statuses PENDING/SUBMITTED/PROVISIONING/RUNNING/TERMINATING/... (runs.py:652-667). +- Run status aggregation: `_analyze_active_run` + `_get_active_run_transition` (`server/background/pipeline_tasks/runs/active.py:182-236, 369-409`): the run is RUNNING as soon as **any** replica job is RUNNING (active.py:393-394). So `RunStatus.RUNNING` alone does NOT mean the model answers requests. +- The real readiness signal is `JobModel.registered` (`server/models.py:578`, "whether the replica is registered to receive service requests"): `_maybe_register_replica` (`jobs_running.py:1119-1159`) sets `registered=True` only for services when `job_num == 0`, probes exist and `is_job_ready(...)` is True; for gateway services it first registers the replica on the gateway (`_register_service_replica`, 1162+; in-server case returns None and just flips the flag). The in-server proxy only routes to `registered == True` jobs (`ServerProxyRepo.get_service`, repo.py:57). +- Rolling deployment: driven by `deployment_num` comparison (`_has_out_of_date_replicas`, active.py:627-642) with surge/teardown in `_build_rolling_deployment_maps` (active.py:645-701); new replicas must become `registered` (probe-ready) before old ones are torn down. `ready_after`/`until_ready` on probes exist specifically for this. + +## 6. Concrete mechanism for the endpoint health-checker + +**Recommended (verified, no new plumbing): replicate `_execute_probe`.** Given the deployed service's RunModel, pick a `JobModel` with `status == JobStatus.RUNNING` (and `job_num == 0`), then: + +```python +from dstack._internal.server.services.jobs.job_replica_http_client import get_service_replica_client +import orjson + +body = orjson.dumps({ + "model": model_name, + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 1, +}).decode() +async with get_service_replica_client(job_model) as client: # SSH tunnel + httpx over UDS + resp = await client.request( + method="post", + url="http://dstack/v1/chat/completions", # or prefix.rstrip('/') + '/chat/completions' + headers=[("Content-Type", "application/json")], + content=body, + timeout=30, + ) + ok = resp.is_success +``` + +This is exactly what probes do today (probes.py:106-126) and works identically for gateway and non-gateway services because it tunnels straight to the replica (bypasses gateway/proxy/auth). Requires the job to have `job_spec.service_port` set (true for all services since 0.19.19; `get_service_port` fallback at `core/models/runs.py:757-762`). `get_service_replica_client(job)` signature: `server/services/jobs/job_replica_http_client.py:22-27`; underlying tunnel `get_service_replica_tunnel` uses `container_ssh_tunnel(job, forwarded_sockets=..., options=...)` (`server/services/ssh.py:81-98`), `SSH_CONNECT_TIMEOUT = timedelta(seconds=10)` (`job_replica_tunnel.py:20`). + +**Even simpler for the endpoint feature**: don't run your own HTTP check at all — rely on the existing probe machinery. If the authored service config has `model:` set (OpenAI format) and `probes` unset, dstack auto-creates the chat-completions probe; the endpoint processor can then just poll `JobModel.registered` (or `is_job_ready(job_model.probes, job_spec.probes)` via `server/services/runs/__init__.py:1226`) to decide active/failed. `run.status == RunStatus.RUNNING` + `job.registered == True` == "model verified to answer /v1/chat/completions". + +**Higher-level typed alternative** (validates the OpenAI response shape): use the proxy-lib stack in-process — `ServerProxyRepo(session).get_model(project, model_name)` + `.get_service(project, run_name)` (`server/services/proxy/repo.py:47,172`), `get_service_replica_client(service, repo, service_conn_pool)` (`proxy/lib/services/service_connection.py:140-163`, pool from `app.state.proxy_dependency_injector` — `ServerProxyDependencyInjector` set in app.py:106, defined in `server/services/proxy/deps.py:11`), then `get_chat_client(model, http_client).generate(ChatCompletionsRequest(model=..., messages=[...], max_tokens=1))` (`proxy/lib/services/model_proxy/model_proxy.py:11`, `clients/openai.py:21-33`, request schema `proxy/lib/schemas/model_proxy.py:11-27`). Caveat: `ServerProxyRepo.get_service` only returns non-gateway, `registered` services — so this path can't health-check gateway-based or not-yet-registered replicas. + +**HTTP path (not recommended for server-internal use)**: `POST {server}/proxy/models/{project}/chat/completions` with `Authorization: Bearer ` — requires a valid project-member token and is marked deprecated in app.py:261. + +## Gotchas +1. Run `RunStatus.RUNNING` fires when ANY replica job is RUNNING (runs/active.py:393-394) — it is NOT "model ready". The readiness signal is `JobModel.registered` (set only after all probes pass, jobs_running.py:1119-1130). An endpoint health-checker that only watches run status would mark endpoints active before the model can serve. +2. `/proxy/models/...` router is mounted with `deprecated=True` (app.py:261) — still works, but new code/URLs should prefer the service-proxy path `/proxy/services///chat/completions` for openai-format models, which is what `_register_service_in_server` now advertises (services/__init__.py:275). +3. The default chat-completions probe is generated ONLY when `model` is `OpenAIChatModel` and `probes` is None (`_probes()`, configurators/base.py:410-419). A bare string `model: ` coerces to OpenAIChatModel, so the common case is covered; TGI-format models get NO default probe. +4. Probes never fail the job — a permanently unhealthy service stays RUNNING with `registered=False` forever. An endpoint feature that wants a "failed" terminal state must add its own timeout on top. +5. `get_service_replica_client` exists TWICE with different signatures: `server/services/jobs/job_replica_http_client.py:22` takes a `JobModel` (context manager, fresh tunnel per call — use this from background tasks) vs `proxy/lib/services/service_connection.py:140` takes `(Service, BaseProxyRepo, ServiceConnectionPool)` (pooled, used by proxy routers). Don't confuse them in the plan. +6. `ServerProxyRepo.get_service`/`list_models` exclude gateway-based runs (`RunModel.gateway_id.is_(None)`) and unregistered jobs — the in-server proxy cannot reach gateway services. The job-tunnel mechanism (probes path) works for both. +7. In probe execution the URL host is literally `"http://dstack"` — a placeholder; transport is a Unix socket. Don't "fix" it. +8. Gateway registration happens synchronously inside `submit_run` (runs/__init__.py:758-760, marked FIXME) and can raise; `complete_service_model` for TGI does a blocking HTTP call to huggingface.co (options.py:25-45). +9. Multi-replica-safe background pattern to copy: `process_probes` (scheduled_tasks/probes.py:29-79) uses `get_locker(get_db().dialect_name).get_lockset(...)` + `with_for_update(skip_locked=True, key_share=True)` + due-based scheduling; scheduled via `AsyncIOScheduler` in `background/scheduled_tasks/__init__.py` (there is also `background/pipeline_tasks/` for the pipeline-style processors). +10. Auth token for HTTP proxy calls is a plain dstack user token of any project member (permissions.py:278-283); there is no separate service/model API key concept. diff --git a/endpoint-implementation-research/settings-migrations-testing.md b/endpoint-implementation-research/settings-migrations-testing.md new file mode 100644 index 0000000000..45fde85fb2 --- /dev/null +++ b/endpoint-implementation-research/settings-migrations-testing.md @@ -0,0 +1,127 @@ +# settings-migrations-testing + +## Summary +Established ground truth for server settings, migrations, models, testing, linting, and packaging conventions in /Users/dstack/dstack. Settings are plain module-level constants in src/dstack/_internal/server/settings.py read via os.getenv/environ helpers with DSTACK_ or DSTACK_SERVER_ prefixes, documented in mkdocs/docs/reference/env.md; feature flags are DSTACK_FF_* on the FeatureFlags class in src/dstack/_internal/settings.py. Migrations are Alembic autogenerate (run from src/dstack/_internal/server with `alembic revision -m "..." --autogenerate`), rendered with render_as_batch=True for SQLite compat, stored under migrations/versions//. Models use DeclarativeBase BaseModel with UUIDType(binary=False) PKs, NaiveDateTime, EnumAsString, EncryptedString custom types, and pydantic-JSON-as-Text columns; multi-replica pipeline rows use PipelineModelMixin (lock_expires_at/lock_token/lock_owner). Test factories live in src/dstack/_internal/server/testing/common.py (not factories.py) with test_db/session fixtures in testing/conf.py; tests run via `uv run pytest src/tests` (add --runpostgres for Postgres). Lint is ruff 0.12.7, types are pyright standard mode; heavy deps are optional-dependencies extras (server, gateway, aws, ...) guarded by try/except ImportError. + +## Key files +- src/dstack/_internal/server/settings.py — module-level constants; JobNetworkMode enum — All server env-var settings; docstring says 'Documented in reference/env.md' +- src/dstack/_internal/settings.py — FeatureFlags, DSTACK_VERSION, DSTACK_RELEASE — Global (non-server) settings; FeatureFlags class holds DSTACK_FF_* flags +- src/dstack/_internal/utils/env.py — Environ.get/get_bool/get_int/get_enum/get_callback; environ singleton — Typed env-var parsing helper used by newer settings +- src/dstack/_internal/server/models.py — BaseModel, PipelineModelMixin, NaiveDateTime, EncryptedString, DecryptedString, EnumAsString, constraint_naming_convention, RunModel, VolumeModel — All ORM models and custom column types +- src/dstack/_internal/server/alembic.ini — — script_location=migrations; file_template puts revisions in /MM_DD_HHMM__.py +- src/dstack/_internal/server/migrations/env.py — — render_as_batch=True (line 82), compare_type=True (line 81), target_metadata=BaseModel.metadata (line 16) +- src/dstack/_internal/server/migrations/versions/2026/03_04_2221_5e8c7a9202bc_add_exports.py — — Recent example migration creating tables with UUIDType PKs, named FKs, batch_alter_table for indexes +- contributing/MIGRATIONS.md — — Alembic command, expand-and-contract rules, one-table-per-migration, CREATE INDEX CONCURRENTLY guidance +- contributing/DEVELOPMENT.md — — uv sync --all-extras; pre-commit; pyright standard mode +- CONTRIBUTING.md — — uv run ruff check --fix; uv run ruff format; uv run pytest src/tests [--runpostgres] +- src/dstack/_internal/server/testing/common.py — create_user, create_project, create_repo, create_run, create_job, create_fleet, create_volume, create_gateway, create_instance, get_run_spec, ComputeMockSpec — THE factory module (testing/factories.py does not exist) +- src/dstack/_internal/server/testing/conf.py — test_db, session, postgres_container fixtures — test_db parametrized sqlite/postgres via indirect=True; postgres uses testcontainers +- src/tests/conftest.py — pytest_addoption(--runpostgres/--runui), disable_feature_flags — Re-exports testing.conf fixtures; autouse fixture forces all FeatureFlags to False +- src/tests/_internal/server/conftest.py — client, test_log_storage, image_config_mock, mock_gateway_connection — Server-level fixtures incl. httpx ASGI client against dstack._internal.server.main:app +- src/tests/_internal/server/background/pipeline_tasks/test_volumes.py — — Canonical pipeline-task test: Fetcher/Worker fixtures, parametrized test_db, factories +- src/tests/_internal/server/background/scheduled_tasks/test_probes.py — — Canonical scheduled-task test: calls process_probes() directly with mocked deps +- pyproject.toml — — ruff/pyright/pytest config, dependency-groups dev, [project.optional-dependencies] extras incl. server/gateway/all +- src/dstack/_internal/server/services/storage/s3.py — — Canonical optional-dependency import pattern (BOTO_AVAILABLE flag + try/except ImportError) +- mkdocs/docs/reference/env.md — — User-facing docs for DSTACK_SERVER_* env vars (server section around lines 94-140) + +## Details +# Conventions cheat-sheet (verified against code, 2026-07-03) + +## 1. Server settings — `src/dstack/_internal/server/settings.py` + +Plain module-level constants evaluated at import time. Module docstring (settings.py:1-3): *"Environment variables read by the dstack server. Documented in reference/env.md"* — the doc file is `mkdocs/docs/reference/env.md` (server env vars listed ~lines 94-140 with `{ #ANCHOR }` markers). + +Naming: env var prefix is `DSTACK_SERVER_*` for server-behavior settings, bare `DSTACK_*` for cross-cutting ones (`DSTACK_DATABASE_URL`, `DSTACK_SENTRY_DSN`, `DSTACK_DB_POOL_SIZE`, `DSTACK_ACME_SERVER`). Python constant name drops the `DSTACK_` prefix (e.g. `SERVER_PORT`, `DATABASE_URL`). + +Declaration patterns (all real examples): +- String w/ default: `SERVER_HOST = os.getenv("DSTACK_SERVER_HOST", "localhost")` (settings.py:28) +- Int: `SERVER_PORT = int(os.getenv("DSTACK_SERVER_PORT", "8000"))` (settings.py:29); `MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25))` (settings.py:54) +- Presence-based bool flag (set-to-anything = true): `SERVER_BACKGROUND_PROCESSING_DISABLED = os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None` followed by `SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED` (settings.py:47-50). Same DISABLED/ENABLED pair pattern for `SERVER_CONFIG_DISABLED/ENABLED` (58-59), `DEFAULT_CREDS_DISABLED/ENABLED` (122-123), `SERVER_SSH_POOL_DISABLED/ENABLED` (152-153). +- Typed helper (`dstack._internal.utils.env.environ`, an `Environ` wrapper over `os.environ` — env.py:12-121, methods `get(name, *, default)`, `get_bool`, `get_int`, `get_enum(name, enum_cls, *, value_type, default)`, `get_callback(name, callback, *, default)`): e.g. `SERVER_METRICS_RUNNING_TTL_SECONDS = environ.get_int("DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS", default=3600)` (settings.py:83-85); enum example `JOB_NETWORK_MODE = environ.get_enum("DSTACK_SERVER_JOB_NETWORK_MODE", JobNetworkMode, value_type=int, default=DEFAULT_JOB_NETWORK_MODE)` (settings.py:178-183). +- Optional str normalized to None: `SERVER_DEFAULT_DOCKER_REGISTRY = os.getenv("DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY") or None` (settings.py:132). +- Dev-only settings live at bottom under a `# Development settings` comment (settings.py:156-165), e.g. `SQL_ECHO_ENABLED`, `SERVER_PROFILING_ENABLED`. + +Feature flags: NOT in server/settings.py. `class FeatureFlags` in `src/dstack/_internal/settings.py:40-52` — env vars of the form `DSTACK_FF_*`, class attrs must be bool, docstring says flags are temporary for large features in development. Current sole flag: `CLI_PRINT_JOB_CONNECTION_INFO = os.getenv("DSTACK_FF_CLI_PRINT_JOB_CONNECTION_INFO") is not None`. In tests, ALL feature flags are force-disabled by the autouse session fixture `disable_feature_flags` in `src/tests/conftest.py:51-62`; to test a flag, monkeypatch `FeatureFlags` per-test. + +## 2. Alembic migrations — `src/dstack/_internal/server/migrations/` + +Generate (contributing/MIGRATIONS.md:7-10): +```shell +cd src/dstack/_internal/server/ +alembic revision -m "" --autogenerate +``` +- `alembic.ini` lives at `src/dstack/_internal/server/alembic.ini`; `script_location = migrations`; `file_template = %%(year)d/%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s` and `recursive_version_locations = true` — so files land in `migrations/versions//MM_DD_HHMM__.py` (e.g. `versions/2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py`). +- `migrations/env.py` uses `target_metadata = BaseModel.metadata` (env.py:16) and configures autogenerate with `compare_type=True, render_as_batch=True` (env.py:81-82). **`render_as_batch=True` is what makes ALTERs SQLite-compatible** — column adds/drops are emitted as `with op.batch_alter_table("
", schema=None) as batch_op:` blocks (see add_gateway_replica_pipeline migration lines 44-65). Plain `op.create_table`/`op.drop_table` used for new tables (add_exports migration lines 24-40). +- Column types in migrations: `sqlalchemy_utils.types.uuid.UUIDType(binary=False)` for UUIDs; `dstack._internal.server.models.NaiveDateTime()` for datetimes (migrations import `import dstack._internal.server.models` for this); `sa.String(length=100)`, `sa.Text()`, `sa.Boolean()`. +- Constraint naming comes from `constraint_naming_convention` (models.py:191-197): `pk_%(table_name)s`, `fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s`, `uq_%(table_name)s_%(column_0_name)s`, `ix_%(column_0_label)s` — migrations pass explicit names via `op.f(...)` (e.g. `name=op.f("fk_exports_project_id_projects")`). +- Multi-replica zero-downtime rules (contributing/MIGRATIONS.md): migrations must not break old replicas; use expand-and-contract for column removals/renames (remove = release 1: `deferred=True` stop reading; release 2: drop). Alter only ONE table per migration/transaction (Postgres ACCESS EXCLUSIVE deadlock risk, MIGRATIONS.md:47-49). Index creation should use `postgresql_concurrently=True` inside `with op.get_context().autocommit_block():`, pre-dropping with `if_exists=True` for retry safety (MIGRATIONS.md:51-75). +- Data backfills inside migrations use lightweight `sa.table(...)/sa.column(...)` partial definitions + `op.execute(sa.update(...))` — see 06_19_0709_857d8fa7fcc5 lines 22-89 (also shows the exact pattern used to retrofit pipeline columns: add nullable, backfill `last_processed_at=created_at` and status via `sa.case`, then `alter_column(..., nullable=False)`). +- `pyproject.toml` [tool.pyright] ignores `src/dstack/_internal/server/migrations/versions`. + +## 3. models.py conventions — `src/dstack/_internal/server/models.py` + +- Base: `class BaseModel(DeclarativeBase): metadata = MetaData(naming_convention=constraint_naming_convention)` (models.py:200-201). SQLAlchemy 2.0 style: `Mapped[...]` + `mapped_column(...)`. +- PK convention: `id: Mapped[uuid.UUID] = mapped_column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4)` (`UUIDType` from `sqlalchemy_utils`). Table names are plural snake_case via `__tablename__` ("runs", "volumes", "gateway_computes"). +- `PipelineModelMixin` (models.py:204-207) — REQUIRED for any row processed by the multi-replica-safe background pipeline machinery: `lock_expires_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)`, `lock_token: Mapped[Optional[uuid.UUID]] = mapped_column(UUIDType(binary=False))`, `lock_owner: Mapped[Optional[str]] = mapped_column(String(100))`. Models using it: RunModel, JobModel, GatewayModel, GatewayComputeModel, FleetModel, InstanceModel, VolumeModel, PlacementGroupModel, ComputeGroupModel, ServiceRouterWorkerSyncModel. Pipeline-processed models additionally carry `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime)` (sometimes `skip_min_processing_interval: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())`) and a partial "fetch queue" index in `__table_args__`, e.g. VolumeModel (models.py:991-998): + ```python + Index("ix_volumes_pipeline_fetch_q", last_processed_at.asc(), + postgresql_where=deleted == false(), sqlite_where=deleted == false()) + ``` + RunModel's variant filters on status: `postgresql_where=status.not_in(RunStatus.finished_statuses())` (models.py:466-471). +- Custom column types (all `TypeDecorator`s defined in models.py): + - `NaiveDateTime` (models.py:57) — impl DateTime; strips tzinfo on write, re-attaches UTC on read. Used for every datetime column; python-side default is `dstack._internal.utils.common.get_current_datetime`. + - `EncryptedString` (models.py:107) — impl String; binds `DecryptedString` pydantic-dual model (models.py:83, has `plaintext`, `decrypted`, `exc`, `get_plaintext_or_error()`); encrypt/decrypt funcs injected via `EncryptedString.set_encrypt_decrypt(...)` (wired by importing `dstack._internal.server.services.encryption` for side effect — see src/tests/_internal/server/conftest.py:9). Used e.g. `UserModel.token: Mapped[DecryptedString] = mapped_column(EncryptedString(200), unique=True)` (models.py:218), `BackendModel.auth = EncryptedString(20000)`. + - `EnumAsString(enum_class, length, fallback_deserializer=None)` (models.py:152) — stores `enum.name` string; used for all status columns: `status: Mapped[VolumeStatus] = mapped_column(EnumAsString(VolumeStatus, 100), index=True)`. +- **No JSON column type.** Structured payloads are pydantic models serialized to `Text`: `run_spec: Mapped[str] = mapped_column(Text)` (models.py:445), `VolumeModel.configuration: Mapped[str] = mapped_column(Text)` (models.py:981), `JobModel.job_spec_data`, etc. Written as `run_spec.json()`, read as `RunSpec.parse_raw(run.run_spec)` (pydantic v1: `pydantic>=1.10.10,<2.0.0` + pydantic-duality). +- FKs: `mapped_column(ForeignKey("projects.id", ondelete="CASCADE"))` + separate `relationship()`. Soft-delete convention: `deleted: Mapped[bool]` (+ optional `deleted_at`, `to_be_deleted` for pipeline-driven deletion). Bools added later use `server_default=false()` (from `sqlalchemy.sql`). +- Docstrings placed under fields as bare strings explain semantics (e.g. models.py:437 `"""`status` must be changed only via `switch_run_status()`."""`). + +## 4. Testing + +- Factories are in `src/dstack/_internal/server/testing/common.py` (**`testing/factories.py` does not exist**; the package has `common.py`, `conf.py`, `matchers.py`). All async, take `session: AsyncSession` first, `session.add(...)` + `await session.commit()`, return the model. Exact names (with a few exact signatures): + - `async def create_user(session, name="test_user", created_at=..., global_role=GlobalRole.ADMIN, token=None, email=None, ssh_public_key=None, ssh_private_key=None, active=True, deleted=False) -> UserModel` (common.py:147) + - `async def create_project(session, owner=None, name="test_project", created_at=..., ssh_private_key="", ssh_public_key="", is_public=False, templates_repo=None, deleted=False) -> ProjectModel` (common.py:200) + - `async def create_repo(...)` (262), `async def create_backend(session, project_id, backend_type=BackendType.AWS, config=None, auth=None, ...)` (228) + - `def get_run_spec(repo_id, run_name="test-run", configuration_path="dstack.yaml", profile=..., configuration=None, ssh_key_pub="user_ssh_key", ...) -> RunSpec` (341) — default configuration is `DevEnvironmentConfiguration(ide="vscode")`; pass e.g. `ServiceConfiguration(...)` to override. + - `async def create_run(session, project, repo, user, fleet=None, gateway=None, run_name=None, status=RunStatus.SUBMITTED, ..., run_spec=None, ...) -> RunModel` (365) — serializes `run_spec.json()` into the Text column. + - `async def create_job(session, run, ...) -> JobModel` (422); `async def create_fleet(...)` (758), `get_fleet_spec` (792), `get_fleet_configuration` (806); `async def create_gateway(...)` (639), `create_gateway_compute` (683); `async def create_instance(...)` (850), `get_instance_offer_with_availability` (964); `async def create_volume(session, project, user, status=..., created_at=..., last_processed_at=..., deleted_at=..., ...)` (1069), `get_volume_configuration` (1150), `get_volume_provisioning_data` (1188); `create_compute_group` (573), `create_placement_group` (1206), `create_probe` (619), `create_secret` (1301), `list_events` (1317), `get_auth_headers(token)` (141). Also `class ComputeMockSpec` (common.py:1404) inheriting all Compute* capability mixins, "Can be used to create Compute mocks that pass all isinstance() asserts". +- DB fixtures — `src/dstack/_internal/server/testing/conf.py`, re-exported in `src/tests/conftest.py` and `src/tests/_internal/server/conftest.py`: + - `test_db` (pytest_asyncio fixture, function-scoped, conf.py:22-55): param "sqlite" (default; in-memory aiosqlite with StaticPool + check_same_thread=False, `BaseModel.metadata.create_all` — **migrations are NOT run in tests**) or "postgres" (testcontainers `PostgresContainer("postgres:16-alpine", driver="asyncpg")`, skipped unless `--runpostgres`; tables created once per session then `TRUNCATE ... RESTART IDENTITY CASCADE` between tests). Calls `override_db(db)` from `dstack._internal.server.db`. + - `session` (conf.py:58-62): `async with test_db.get_session() as session: yield session`. +- Background-task test pattern (pipeline): `src/tests/_internal/server/background/pipeline_tasks/test_volumes.py` — class-level decorators: + ```python + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + class TestVolumeFetcher: + async def test_...(self, test_db, session: AsyncSession, fetcher: VolumeFetcher): ... + ``` + Fixtures construct pipeline components directly: `VolumeWorker(queue=Mock(), heartbeater=Mock(), pipeline_hinter=Mock())`, `VolumeFetcher(queue=asyncio.Queue(), queue_desired_minsize=1, min_processing_interval=timedelta(seconds=15), lock_timeout=timedelta(seconds=30), heartbeater=Mock())`; tests seed rows with factories, then `items = await fetcher.fetch(limit=10)` / call worker methods; lock behavior asserted via `lock_token/lock_expires_at/lock_owner`. Imports come from `dstack._internal.server.background.pipeline_tasks.volumes` (`VolumeFetcher, VolumePipeline, VolumePipelineItem, VolumeWorker`). Test dirs: `src/tests/_internal/server/background/pipeline_tasks/` (test_fleets.py, test_gateways.py, test_gateway_replicas.py, test_volumes.py, test_runs/, test_instances/, ...) and `.../scheduled_tasks/` (test_probes.py, test_events.py, ...). Scheduled-task tests call the task function directly (e.g. `from ...background.scheduled_tasks.probes import process_probes`; `pytestmark = pytest.mark.usefixtures("image_config_mock")`; freezegun `freeze_time` used). + Note: pytest-asyncio runs in default (strict) mode — every async test/class needs explicit `@pytest.mark.asyncio`; no `asyncio_mode` is set in pyproject. +- Other server fixtures (src/tests/_internal/server/conftest.py): `client` (httpx.AsyncClient over `ASGITransport(app=dstack._internal.server.main.app)`), `image_config_mock`, `test_log_storage`, `mock_gateway_connection`. Network is blocked by default: pytest addopts `--disable-socket --allow-hosts=127.0.0.1,localhost --allow-unix-socket` (pyproject.toml:117-122); pytest-env sets `DSTACK_SSHPROXY_API_TOKEN=test-token`, `DSTACK_CLI_RICH_FORCE_TERMINAL=0`. +- Running tests (CONTRIBUTING.md:49-64): `uv run pytest src/tests`; `uv run pytest src/tests --runpostgres` for Postgres. CI (build-artifacts.yml:93) runs `uv run pytest -n auto src/tests --runui $RUNPOSTGRES`. Custom options defined in src/tests/conftest.py: `--runui`, `--runpostgres`; markers `ui`, `postgres`, `windows`, `windows_only` (+ `shim_version`, `dockerized` in pyproject). + +## 5. Linting / type-checking + +- ruff (pyproject.toml:84-96): `target-version = "py310"`, `line-length = 99`, lint select `["E","F","I","Q","W","PGH","FLY","S113"]`, ignore `["E501","E712"]`, isort `known-first-party = ["dstack"]`. Pinned `ruff==0.12.7` in dev group ("should match .pre-commit-config.yaml"); pre-commit hooks `ruff` + `ruff-format` from astral-sh/ruff-pre-commit. Commands per CONTRIBUTING.md: `uv run ruff check --fix` then `uv run ruff format`. +- pyright (pyproject.toml:98-113): `typeCheckingMode = "standard"`; `include` is a whitelist — `src/dstack/plugins`, `src/dstack/_internal/server`, `src/dstack/_internal/core/services`, backends aws/kubernetes/runpod, cli configurators/commands, and `src/tests/_internal/server/background/pipeline_tasks`; ignores `src/dstack/_internal/server/migrations/versions`. CI runs `jakebailey/pyright-action@v3` after `uv sync --all-extras` (build-artifacts.yml:74-79). Local: `uv tool install pyright && pyright -p .` (DEVELOPMENT.md). No mypy. +- requires-python `>=3.10`; pydantic is v1 (`pydantic>=1.10.10,<2.0.0` + `pydantic-duality>=1.2.4`). + +## 6. Optional dependencies / extras (pyproject.toml:172-271) + +- Base `[project.dependencies]` is the lightweight CLI/client set (pyyaml, requests, paramiko, rich, pydantic v1, gpuhunt, apscheduler<4, ...). Build backend: hatchling. +- `[project.optional-dependencies]`: `gateway` (fastapi/starlette/uvicorn/httpx/jinja2/aiorwlock/aiocache), `server` (fastapi, starlette, uvicorn[standard], sqlalchemy[asyncio]>=2.0.0, sqlalchemy_utils, alembic>=1.16.0, aiosqlite, asyncpg, alembic-postgresql-enum, sentry-sdk[fastapi], prometheus-client, grpcio, protobuf, smg-grpc-proto==0.4.9, docker, python-dxf, httpx, jinja2, watchfiles, requests-unixsocket, python-json-logger), then per-backend extras that each depend on `dstack[server]`: `aws` (boto3/botocore), `azure`, `gcp`, `datacrunch`, `verda`, `kubernetes`, `lambda`, `oci`, `nebius`, `fluentbit` (fluent-logger + elasticsearch), `crusoe` (server only), and `all = dstack[gateway,server,aws,azure,gcp,verda,kubernetes,lambda,nebius,oci,crusoe,fluentbit]`. Dev tooling is in `[dependency-groups] dev` (pytest~=8.0, pytest-asyncio, pytest-xdist, freezegun, testcontainers, openai>=1.68.2 which is dev-only for gateway/OpenAI-compat tests, ruff, ...) — installed via `uv sync --all-extras` (installs extras + dev group). +- Runtime guard pattern for optional imports — module-level availability flag, class defined only when import succeeds (src/dstack/_internal/server/services/storage/s3.py:5-13): + ```python + BOTO_AVAILABLE = True + try: + import botocore.exceptions + from boto3 import Session + except ImportError: + BOTO_AVAILABLE = False + else: + class S3Storage(BaseStorage): ... + ``` + Same pattern in services/storage/gcs.py, services/logs/aws.py, services/logs/gcp.py, services/logs/fluentbit.py, services/plugins.py. An `anthropic` extra would follow this: new extra in [project.optional-dependencies] depending on `dstack[server]`, guarded import + availability flag, and inclusion in `all`. **No anthropic/openai runtime dependency currently exists in project dependencies or extras** (openai appears only in the dev group). + +## Gotchas +1) `src/dstack/_internal/server/testing/factories.py` DOES NOT EXIST — factories are in `testing/common.py`; importing "factories" would be a hallucination. 2) Tests create schema via `BaseModel.metadata.create_all`, NOT alembic — a new model works in tests without a migration, so a missing migration won't fail unit tests; the migration must still be authored for real deployments. 3) Migrations must be multi-replica/zero-downtime safe (expand-and-contract; only one table altered per migration; concurrent index creation with if_exists pre-drop) per contributing/MIGRATIONS.md — a naive autogenerated migration may violate this. 4) pytest runs with `--disable-socket` (only localhost allowed) — any Anthropic API client code in tests must be fully mocked, no network. 5) pytest-asyncio is in strict mode: every async test needs `@pytest.mark.asyncio`; Postgres coverage requires `@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)` and is skipped without `--runpostgres`. 6) pydantic is v1 (<2.0.0) with pydantic-duality — do not plan pydantic v2 APIs (`model_dump`, `field_validator`); persisted specs use `.json()`/`.parse_raw()`. 7) There is no JSON column type in models.py — structured data goes in `Text` columns as pydantic JSON. 8) Feature flags live in `dstack._internal.settings.FeatureFlags` (DSTACK_FF_*), not server/settings.py, and are auto-disabled in all tests by an autouse fixture — tests of a flag must monkeypatch FeatureFlags. 9) pyright only checks whitelisted paths (include list) — new dirs under `_internal/server` are covered automatically, but new test dirs are NOT unless added (only `src/tests/_internal/server/background/pipeline_tasks` is type-checked today). 10) `settings.py` env vars must be documented in `mkdocs/docs/reference/env.md` (settings.py module docstring says so; some defaults note 'keep in sync' with that doc). 11) Alembic autogenerate must be run from `src/dstack/_internal/server/` (alembic.ini lives there); revision files are placed in a year subdirectory automatically by file_template. 12) New pipeline-style tables need PipelineModelMixin columns + `last_processed_at` + a partial `ix_
_pipeline_fetch_q` index with BOTH `postgresql_where` and `sqlite_where`. 13) The `anthropic` package is not a dependency anywhere; `openai` exists only in the dev dependency group — an Anthropic SDK dep should be added as a new optional extra depending on dstack[server] with a try/except ImportError guard (S3Storage pattern). diff --git a/endpoint-implementation-research/skills-and-pr3856.md b/endpoint-implementation-research/skills-and-pr3856.md new file mode 100644 index 0000000000..713be189bf --- /dev/null +++ b/endpoint-implementation-research/skills-and-pr3856.md @@ -0,0 +1,85 @@ +# skills-and-pr3856 + +## Summary +PR #3856 ("Add dstack-dev skill for using dev environments", author peterschmidt85) is OPEN and NOT merged (mergedAt=null, closedAt=null, no reviews, CI green; it was once stale-closed by a bot and reopened, last updated 2026-07-03). It adds a new skills/dstack-dev/SKILL.md (39 lines, a playbook for using dev environments + SSH to experiment/tune/debug workloads before promoting them to tasks/services) and appends 2 description lines to skills/dstack/SKILL.md. The local checkout has exactly one skill, skills/dstack/SKILL.md (567 lines), a Claude Code-format skill (YAML frontmatter name/description + markdown body, single file, no referenced files) teaching agents dstack YAML authoring for all 6 config types, exact CLI workflows (apply/-d/-y, ps, attach, offer, logs, stop), anti-hallucination rules, and service/model-endpoint details. Other agent-facing material: raw .md docs mirrored to the site (https://dstack.ai/docs/**.md), generated llms.txt (~5KB) and llms-full.txt (~482KB), inference example docs with ready-made service YAMLs (vllm/sglang/trtllm/nim/dynamo), and a published skill at https://dstack.ai/skill.md + /.well-known/skills/. Critically, NONE of this ships in the installed dstack wheel (skills/, mkdocs/, examples/ are all outside src/dstack; hatchling only packages src/dstack plus the gitignored statics via `artifacts`), and there are zero "anthropic"/"claude"/skill-loading references in src/dstack — a server-embedded agent needs its instructional context vendored into src/dstack/... (git-tracked files there ship automatically) or fetched over the network. + +## Key files +- skills/dstack/SKILL.md — frontmatter name=dstack; sections: Quick agent flow, Agent execution guidelines, Configuration types, Essential CLI commands, Troubleshooting — The only skill in the local checkout (567 lines). Claude Code skill format: YAML frontmatter (name: dstack; description: one-liner) + markdown body. Teaches: config types with YAML examples (dev-environment, task, service, fleet, volume, gateway), agent CLI workflows (echo "n" | dstack apply -f to preview; dstack apply -f -y -d to submit; dstack ps -v; background nohup dstack attach), anti-hallucination rules ('If a command or flag isn't documented, it doesn't exist'), service endpoints/auth/model base_url, troubleshooting, and links to raw-.md doc URLs plus https://dstack.ai/llms-full.txt. +- scripts/docs/hooks.py — SKILL_PATH=("skills","dstack","SKILL.md") (line 22), on_post_build (line 256), _write_well_known_skills (line 295), _generate_llms_files (line 342) — mkdocs hooks (registered in mkdocs.yml:19-20). on_post_build (line 256) copies every docs .md raw into the built site (so every page is fetchable as https://dstack.ai/docs/.md), _write_well_known_skills (line 295) publishes ONLY skills/dstack/SKILL.md (SKILL_PATH constant, line 22) to site root skill.md/SKILL.md and .well-known/skills/dstack/SKILL.md + index.json, _generate_llms_files (line 342) runs gen_llms_files.py. dstack-dev from PR #3856 would NOT be web-published without changing SKILL_PATH handling. +- scripts/docs/gen_llms_files.py — generate_llms_files(repo_root, site_dir, mkdocs_config) (line 224), INCLUDE_SECTIONS/EXCLUDE_SECTIONS (lines 15-16) — Generates llms.txt (nav-derived link index with descriptions) and llms-full.txt (raw concatenation of all included .md sources). INCLUDE_SECTIONS = ["Getting started", "Concepts", "Guides", "Examples"]; EXCLUDE_SECTIONS = ["Reference"] (line 15-16) — so the YAML schema reference pages are NOT in llms-full.txt. Local build artifacts: site/llms.txt (5,372 bytes), site/llms-full.txt (482,113 bytes); site/ is gitignored build output. +- pyproject.toml — [tool.hatch.build.targets.wheel] artifacts (line 65-68) — hatchling build backend (lines 50-52). Wheel/sdist only add artifacts "src/dstack/_internal/server/statics/**" (lines 60-68) beyond the auto-detected src/dstack package — `artifacts` exists solely because statics/ is gitignored (.gitignore line 26). skills/, mkdocs/, examples/ are repo-root dirs outside src/ and are NOT in the installed package. Git-tracked non-.py files inside src/dstack DO ship (e.g. nginx jinja templates, alembic.ini), so vendoring a committed .md under src/dstack/... ships automatically with no pyproject change. +- mkdocs/docs/installation.md — — Lines 208-222: 'Install agent skills' section — the documented consumption path is `npx skills add dstackai/dstack` via skills.sh (https://skills.sh/dstackai/dstack/dstack), for 'AI agents like Claude, Codex, and Cursor ... to help them use the CLI and edit configuration files'. +- mkdocs/docs/examples/inference/vllm.md — — Example of ready-made agent context for authoring service configs: full `type: service` YAML deploying Qwen/Qwen3.6-27B with vllm/vllm-openai:v0.19.1 on H100:4 (NVIDIA and AMD ROCm tabs), /root/.cache instance volume, model:/port: fields. Siblings: sglang.md, trtllm.md, nim.md, dynamo.md under mkdocs/docs/examples/inference/. All included in llms-full.txt (Examples section is in INCLUDE_SECTIONS). Served raw at https://dstack.ai/docs/examples/inference/vllm.md. +- AGENTS.md — — Repo-root contributor guidelines for agents working ON the dstack codebase (uv sync, uv run pytest, ruff, pyright, style rules). Not about using dstack to deploy; not packaged; not relevant as service-authoring context. +- mkdocs.yml — — site_url https://dstack.ai (line 3), docs_dir: mkdocs (line 17), hooks: scripts/docs/hooks.py (lines 19-20). Nav 'More' section links llms-full.txt and skill.md as external URLs (lines 397-398). + +## Details +## 1. PR #3856 — state and content + +`gh pr view 3856 --repo dstackai/dstack`: +- Title: "Add dstack-dev skill for using dev environments"; author `peterschmidt85` (Andrey Cheptsov); base `master`, head `skills-dstack-dev` (branch exists on origin, tip 7c2f5a034d10a541e4212b4599d9f5b13b04c915). +- **State: OPEN. mergedAt: null. closedAt: null. NOT merged — confirmed.** Created 2026-05-06, updated 2026-07-03. Bot comments show it was stale-marked and stale-closed once, then reopened. reviewDecision empty, zero reviews, mergeable=MERGEABLE, CI checks SUCCESS. +- Files: `skills/dstack-dev/SKILL.md` (+39 new) and `skills/dstack/SKILL.md` (+2). + +### skills/dstack-dev/SKILL.md content (from `gh pr diff 3856`) +Frontmatter: `name: dstack-dev`; description says it is "specifically designed for using dstack's dev environments when it's required to experiment with code, tune and debug workloads before running them as tasks or services". +Body — "Using dev environments for experimenting, tuning, and debugging code": +1. Enable the base `dstack` skill first (explicit cross-reference). +2. **Fleet**: use `dstack offer` to pick instance config (GPU count/name, disk sized for model weights); ensure/create a matching fleet; VM-based backends allow pre-provisioned `nodes: ` and instance reuse; container backends need `nodes: 0..` ranges; SSH fleets for on-prem; prefer VM-based when possible. +3. **Docker image**: pick the image you plan to use later in the task/service; mount optional instance volume at `/root/.cache` if the workload downloads models/kernels; `docker: true` (VM-based fleets only) to try multiple images via `docker run` in one run. +4. **Dev environment**: author dev-environment config pinned to the fleet, `dstack apply -d`. +5. **SSH**: once `running`, `dstack attach --logs`, then `ssh ` to run commands interactively; **both must run outside the sandbox or they fail**. +6. **Iterate**: stop run, change image/fleet, re-apply. +Then an "Example: inference" 10-step recipe: research framework/GPU/disk requirements from official docs/model card → `dstack offer` → fleet → image (+cache volume / `docker: true`) → dev env → attach → ssh + experiment → reuse the proven hardware+commands in a task or **service** configuration. + +This is directly the "agent prototypes on a dev environment, then promotes to a service" workflow. Because the PR is unmerged, `skills/dstack-dev/` **does not exist** in the local checkout (`ls /Users/dstack/dstack/skills/` shows only `dstack/`). + +### skills/dstack/SKILL.md change in the PR +Adds only 2 lines to the frontmatter description: "This skill covers dstack's core capabilities for managing fleets, dev environments, tasks, and services. It includes best practices for applying configurations, monitoring workloads, and troubleshooting common issues." The local checkout matches the pre-PR version (description is the single one-liner, `skills/dstack/SKILL.md:3-4`). + +## 2. Current skills/ directory (local checkout, master) + +`/Users/dstack/dstack/skills/` contains exactly one skill: `skills/dstack/SKILL.md` (567 lines). No other files, no `references/` subdir, no scripts. Detailed content: + +- **Frontmatter** (lines 1-5): `name: dstack`, multi-line `description` (one sentence). +- **Overview / How it works** (lines 9-31): server (local/remote/dstack Sky), CLI (project config in `~/.dstack/config.yml`, managed with `dstack project`), `*.dstack.yml` config files; `dstack apply` plan-then-submit semantics, attach-by-default, `-d` for detached. +- **Quick agent flow** (lines 33-39): `echo "n" | dstack apply -f ` to preview plan → `dstack apply -f -y -d` → `dstack ps -v` → attach in background if needed → escalate if sandboxed attach fails. +- **Anti-hallucination rules** (lines 41-52): "Never propose `dstack` CLI commands or YAML syntaxes that don't exist"; verify via `--help`; never invent flags; never guess YAML property names; never run `dstack apply` for runs without `-d` in automated contexts; don't reformat tabular CLI output; prefer `-y` over `echo "y" |`. +- **Agent execution guidelines** (lines 54-141): output accuracy, `--help` verification, blocking commands (`dstack attach`, `apply` without `-d`, `ps -w`), 10-60s timeouts otherwise, confirmation handling, detached-run follow-up, background attach pattern (`nohup dstack attach --logs > /tmp/.attach.log 2>&1 & echo $! > /tmp/.attach.pid`), sandbox-permission escalation guidance, "dstack run only supports `dstack run get --json`". +- **Configuration types** (lines 143-340): common params (repo/repos/files/image/docker:true/env/volumes/resources); best practices (always set `name`; env var *names* only with values in `.envrc`; `python` and `image` mutually exclusive); `files`/`repos` intent policy; then per-type YAML examples: dev-environment, task (ports, distributed `nodes`), **service** (vLLM example with `port: 8000`, `model: meta-llama/...`, `resources.gpu: 80GB, disk: 200GB`; endpoint URL schemes `/proxy/services///` and `https://./`; `Authorization: Bearer ` unless `auth: false`; model endpoint via `service.model.base_url` from `dstack run get --json`, OpenAI-compatible = `service.url` + `/v1`; curl chat/completions example), fleet (nodes ranges, `placement: cluster`, ssh_config), volume (network + instance-volume `/root/.cache` pattern with `optional: true`), gateway (when required: autoscaling, rate limits, HTTPS custom domain, WebSockets). +- **Essential CLI commands** (lines 342-503): apply workflows for run vs infra configs; `dstack fleet [get|delete -i]`; `dstack ps [-v|--json|-n]`; `dstack run get --json`; `dstack attach [--logs]`; `dstack logs [-d|--replica|--job]`; `dstack stop [-y|--abort]`; `dstack offer` with `--backend/--gpu/--fleet/--max-offers/--group-by gpu|backend|region|count/--json`. +- **Troubleshooting + resources** (lines 505-566): JSON-inspection recipes; common issues (No offers/No fleet/config errors/provisioning timeouts); links exclusively to raw-.md doc URLs (e.g. `https://dstack.ai/docs/concepts/services.md`, `https://dstack.ai/docs/reference/dstack.yml/service.md`, `https://dstack.ai/docs/guides/troubleshooting.md`) and "Full documentation: https://dstack.ai/llms-full.txt" (line 566). + +Git history of the file: last touched by #3869 (CLI & API guide rework) and #3859 (docs/ → mkdocs/ rename); substantive skill edits in #3774 (offer --fleet), #3634, #3560, #3555, #3554, #3547. + +## 3. How the skills are consumed + +- **Format**: standard agent-skill (Claude Code) format — YAML frontmatter with only `name` and `description`, markdown body. Single self-contained SKILL.md; `.well-known/skills/index.json` lists `"files": ["SKILL.md"]` — no referenced auxiliary files. +- **Install channel (documented)**: `npx skills add dstackai/dstack` via skills.sh — `mkdocs/docs/installation.md:208-222` ("Install agent skills", links https://skills.sh/dstackai/dstack/dstack), cross-referenced from `quickstart.md:9` and `guides/cli-api.md:9`. This reads the GitHub repo's `skills/` directory. +- **Web publication**: `scripts/docs/hooks.py` at docs build: `_write_well_known_skills` (hooks.py:295-339) parses `skills/dstack/SKILL.md` frontmatter and copies it to site root as both `skill.md` and `SKILL.md`, plus `.well-known/skills/dstack/SKILL.md` and `.well-known/skills/index.json`. Verified in local build output `site/.well-known/skills/index.json`. Note `SKILL_PATH = ("skills", "dstack", "SKILL.md")` (hooks.py:22) is hard-coded to the single dstack skill — merging PR #3856 would NOT auto-publish dstack-dev to the website (only to the git repo / skills.sh path). +- **This session**: a `dstack` skill is also loaded in the current agent environment (Skill tool list), with the same one-line description as the local SKILL.md frontmatter. + +## 4. Other agent-facing material a server-embedded agent could use + +- **llms.txt / llms-full.txt**: generated at docs build by `scripts/docs/gen_llms_files.py` (invoked from hooks.py:342-365). `llms.txt` = nav-driven link index (title + frontmatter description per page, URLs ending in `.md`). `llms-full.txt` = raw concatenation of all pages in sections Getting started, Concepts, Guides, Examples (`INCLUDE_SECTIONS`, gen_llms_files.py:15); the Reference section (dstack.yml schema reference, HTTP API) is EXCLUDED (line 16). Local build: `site/llms.txt` 5,372 B; `site/llms-full.txt` 482,113 B (~120k tokens — too big to inline whole into a prompt; would need slicing). Live at https://dstack.ai/llms.txt and https://dstack.ai/llms-full.txt. `site/` is gitignored build output — not a source artifact. +- **Raw markdown docs endpoints**: hooks.py `on_post_build` (256-292) copies every `mkdocs/docs/**/*.md` verbatim into the built site (with `#SCHEMA#` placeholders in `docs/reference/**` expanded via `gen_schema_reference.py`), and `mimetypes.add_type("text/plain", ".md")` (hooks.py:17). So every doc page, including the full YAML config reference `docs/reference/dstack.yml/service.md`, is fetchable as plain markdown from dstack.ai. +- **Inference examples with ready service YAMLs**: `mkdocs/docs/examples/inference/{vllm,sglang,trtllm,nim,dynamo}.md` — each contains complete `type: service` configs for a concrete model (vllm.md: Qwen/Qwen3.6-27B, image vllm/vllm-openai:v0.19.1, H100:4, NVIDIA + AMD tabs, cache instance volume). Index page `mkdocs/docs/examples.md`. Source-code examples live in repo `examples/` (distributed-training, llms, misc, models, plugins, server-deployment, single-node-training). These example doc pages ARE included in llms-full.txt. +- **AGENTS.md** (repo root): codebase-contributor guidelines (uv, pytest, ruff, style) — for agents developing dstack itself, not for authoring service configs. + +## 5. Installed package vs vendoring (key packaging facts) + +- Build backend is hatchling (pyproject.toml:50-52); the wheel auto-detects `src/dstack` and the only extra `artifacts` entry is `src/dstack/_internal/server/statics/**` (pyproject.toml:60-68) — needed because statics is gitignored (.gitignore:26) and hatchling excludes VCS-ignored files by default. +- **`skills/`, `mkdocs/`, `examples/`, `site/llms*.txt` are all repo-root paths outside `src/` and are NOT part of the installed `dstack` package.** A server-embedded agent cannot `importlib.resources`-load any of today's skill/doc material from a pip-installed dstack. +- Precedent for vendoring: git-tracked non-.py files inside `src/dstack` do ship in the wheel today (e.g. `src/dstack/_internal/proxy/gateway/resources/nginx/*.jinja2`, `src/dstack/_internal/core/backends/template/*.jinja`, `src/dstack/_internal/server/alembic.ini`). So committing e.g. `src/dstack/_internal/server/resources/agent/SKILL.md` (or prompt .md files) ships automatically with no pyproject change; only gitignored/generated files would need an `artifacts` entry. +- **No existing agent plumbing in the server**: `grep -ri anthropic src/dstack --include=*.py` → 0 hits; no "claude" hits; no code in `src/dstack/_internal/server` references skills or SKILL.md. The `openai` package appears only in the dev dependency group (pyproject.toml:149) for tests. Any Anthropic API client, prompt loading, or skill embedding is net-new work. +- Alternative to vendoring: fetch https://dstack.ai/skill.md or https://dstack.ai/llms-full.txt at runtime — but that adds a network/version-skew dependency, and note the CLI-driving portions of SKILL.md (attach/nohup/sandbox guidance) target an interactive human-machine CLI agent, not a server-side API-driven agent; a server-embedded agent would author configs via the Python API/HTTP API, so only the YAML-authoring sections (Configuration types, service endpoint semantics) and the inference example YAMLs are directly reusable context. + +## Gotchas +1. PR #3856 is OPEN, not merged — `skills/dstack-dev/` does not exist on master or in the local checkout; do not reference it as available. It also has zero reviews and was stale-closed once (reopened 2026-07-03), so do not assume imminent merge. +2. The skills are NOT in the installed Python package. Nothing under repo `skills/`, `mkdocs/`, or `examples/` ships in the dstack wheel (hatchling packages only src/dstack + gitignored statics via `artifacts`). A plan that says "server loads skills/dstack/SKILL.md from the package" is wrong — it must vendor content under src/dstack (git-tracked files there ship automatically) or fetch from dstack.ai. +3. There is zero existing Anthropic/agent/skill-loading code in src/dstack (grep confirms 0 hits for "anthropic"); an anthropic SDK dependency would also be net-new (not in any dependency group). +4. skills/dstack/SKILL.md is written for a CLI-driving interactive agent (sandbox escalation, nohup attach, `echo "n" | dstack apply`). A server-embedded agent uses the API, so only parts of it (config-type YAML sections, service endpoint/auth/model-URL semantics) transfer; the CLI-workflow sections would be misleading context. +5. llms-full.txt is ~482KB and EXCLUDES the Reference section — the authoritative dstack.yml property reference (docs/reference/dstack.yml/service.md) is only available as a separately-fetched raw .md URL (schema-expanded at build time), not inside llms-full.txt. +6. hooks.py hard-codes SKILL_PATH to the single dstack skill; the website publishing path (dstack.ai/skill.md, .well-known/skills) covers only that one skill even after #3856 merges. +7. site/ (containing built llms.txt/llms-full.txt/skill.md) is gitignored build output — these files exist only after a docs build; don't treat site/ paths as source artifacts. +8. Recent renames: docs/ was renamed to mkdocs/ and examples moved under mkdocs/docs/examples (PR #3859); older references to a top-level docs/ directory are stale. diff --git a/endpoint-implementation-research/verify-findings.md b/endpoint-implementation-research/verify-findings.md new file mode 100644 index 0000000000..a2de55e57a --- /dev/null +++ b/endpoint-implementation-research/verify-findings.md @@ -0,0 +1,147 @@ +# Verifier 0 + +OVERALL: The plan's sections 2-5 are exceptionally well-grounded: every file path, symbol name, and cited line number I checked (configurations.py:1316/1384/1393/1401/1440, models.py:204/951, volumes.py:259, events.py:12, services/events.py:89, jobs_running.py:1158, probes.py:106, plan.py:970, configurators/__init__.py:27) resolves exactly as claimed, and the mechanism descriptions (lock dance, pipeline fetch/lease, pydantic-duality rules, ProjectMember vs ProjectAdmin, alembic conventions) match the code verbatim. I found no blockers or majors — only four minor corrections: `registered` is in fact set to False during job termination (jobs_terminating.py:787), so 'only ever set to True' overstates it (though the probe-failure conclusion holds); `_get_job_plan` also requires `profile.instances is None` to add cloud offers; MIGRATIONS.md has no literal 'one table per migration' rule; and the proposed `delete_endpoints` signature omits the `user` param that the volumes precedent needs for event emission (and delete_volumes does not hint the pipeline). External-URL claims (recipes.vllm.ai, docs.sglang.io) were not network-verified but everything checkable locally, including PR #3856 being OPEN, was confirmed. + +## [minor] [PARTIALLY_WRONG] 2. Ground truth, item 5 +CLAIM: `registered` is never unset by later probe failures (verified: it's only ever set to `True` at `jobs_running.py:1158`) +CORRECTION: The parenthetical is false: `registered` IS also assigned `False` at src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py:787 in `_unregister_replica_and_update_result` (result.job_update_map["registered"] = False) when a replica is unregistered during job termination. The load-bearing conclusion survives — probe failures never unset it (the True assignment at jobs_running.py:1158 in `_maybe_register_replica` is the only place it is set True, and the False path only runs in the terminating pipeline) — but the plan should not claim it is 'only ever set to True'. + +## [minor] [PARTIALLY_WRONG] 2. Ground truth, item 7 (and §7.3 step 2) +CLAIM: plan offers always include existing-instance offers, and add new-cloud offers only when profile.creation_policy == CreationPolicy.REUSE_OR_CREATE +CORRECTION: The actual condition in `_get_job_plan` (src/dstack/_internal/server/services/runs/plan.py:979) is `if profile.creation_policy == CreationPolicy.REUSE_OR_CREATE and profile.instances is None:` — backend offers are also suppressed when `profile.instances` is set, even under REUSE_OR_CREATE. The direction the plan relies on (creation_policy=reuse ⇒ offers are exclusively existing-instance offers) is correct, but 'non-empty offers ⇔ matches existing fleets' as stated omits the `instances` clause. + +## [minor] [PARTIALLY_WRONG] 4. DB model, migration, events — Migration bullet +CLAIM: Respect contributing/MIGRATIONS.md (one table per migration; this one is additive so zero-downtime-safe) +CORRECTION: contributing/MIGRATIONS.md contains no 'one table per migration' rule. The actual rule (section 'Altering multiple tables') is: 'Altering multiple tables should be done in separate transactions/migrations' — it applies to ALTERs (ACCESS EXCLUSIVE locks), not to creating a new table. The doc also recommends CREATE INDEX CONCURRENTLY for adding indexes (relevant only for existing tables). The plan's conclusion (a single additive create_table migration is zero-downtime-safe) is unaffected. The autogenerate command and versions// layout are exactly right (alembic.ini file_template = %(year)d/%(month).2d_... at src/dstack/_internal/server/alembic.ini:11). + +## [minor] [PARTIALLY_WRONG] 5.1 Service module — delete_endpoints +CLAIM: delete_endpoints(session, project, names): volumes-style lock-retry (10×) ...; set to_be_deleted=True; emit; commit; hint +CORRECTION: Two deltas vs the cited precedent `delete_volumes` (src/dstack/_internal/server/services/volumes.py:321): (1) its signature is `(session, project, names, user)` — `user` is required because the event is emitted with `events.UserActor.from_user(user)` (volumes.py:373); the plan's signature omits `user` yet mandates an emit. (2) `delete_volumes` does NOT hint the pipeline after commit (no pipeline_hinter parameter); adding a hint for endpoints is a reasonable improvement but is not 'volumes-style'. The lock-retry (range(10), lock_expires_at.is_(None), with_for_update(key_share=True), ServerClientError on failure, to_be_deleted=True) is confirmed verbatim. + +## [minor] [PARTIALLY_WRONG] 2. Ground truth, item 10 +CLAIM: The existing skills/dstack/SKILL.md (567 lines) targets a CLI-driving interactive agent +CORRECTION: skills/dstack/SKILL.md is 566 lines (wc -l; file ends with a trailing newline). PR #3856 being OPEN (not merged) and skills/dstack-dev not existing on master were both confirmed (gh pr view 3856 → state OPEN; no skills/dstack-dev directory). + +# Verifier 1 + +OVERALL: The pipeline-framework parts of the plan are accurate: the Pipeline/Fetcher/Worker/Heartbeater contracts, the volume fetch query it copies (including the last_processed_at == created_at fast path, lock_owner filter, and with_for_update(skip_locked=True, key_share=True, of=Model)), the InstanceFetcher per-status-interval precedent (instances/__init__.py:190-211 uses min_processing_interval * 2 for steady-state statuses, so the plan's ACTIVE-interval WHERE clause fits), the update-map helpers, and all section-9 settings/packaging/docs facts check out against the code. The one substantive error is the run-teardown shortcut: delete_runs raises ServerClientError for any non-finished run and stop_runs only flips the run to TERMINATING for asynchronous termination by RunPipeline, so the agent-path pre-cleanup in 6.2 step 3 (and the 8.3 stop_service tool) written as a one-shot 'stop_runs + delete_runs' will fail on active leftovers and, combined with the attempts-before-launch rule, can exhaust the attempt budget without ever running the agent — it must adopt the same two-iteration wait-until-finished pattern the plan already prescribes for _process_to_be_deleted_endpoint. Two nuances worth carrying into implementation: the interim token-guarded attempts UPDATE is mechanically consistent with the framework (it leaves lock_token/lock_expires_at untouched so the Heartbeater and final write are unaffected) but has no exact precedent — existing workers only commit mid-process for related-row lock claims (fleets.py, jobs_submitted.py), so the final update map must be careful not to write a stale attempts value over the interim one; and the 6.4 probe helper must replicate _execute_probe's except (SSHError, httpx.RequestError) → False clause or health-check failures will raise past the failure counter. + +## [major] [PARTIALLY_WRONG] 6.2 _process_provisioning_endpoint step 3 (agent pre-cleanup); also 8.3 stop_service tool +CLAIM: If a leftover run named after the endpoint exists (from a crashed attempt), terminate it first (stop_runs(abort=True) + delete_runs) so the agent starts clean. +CORRECTION: delete_runs cannot be called right after stop_runs when the leftover run is still active. src/dstack/_internal/server/services/runs/__init__.py::delete_runs (line 909) raises ServerClientError 'Cannot delete active runs: ...' (lines 932-936) for any run not in RunStatus.finished_statuses(); stop_runs (line 865) only sets termination_reason and switches the run to TERMINATING ('The run will be terminated by RunPipeline', line 903) — actual termination is asynchronous. TERMINATING is not a finished status (core/models/runs.py::RunStatus.finished_statuses = [TERMINATED, FAILED, DONE]). The one-shot sequence works only if the leftover run already finished; for an active leftover it raises, the exception is swallowed by Worker.start's generic handler (base.py:360-361), the final update-map write is skipped, and — since attempts is persisted before the agent launches — each such loop can burn an attempt without ever running the agent. Fix: mirror _process_to_be_deleted_endpoint's own (correct) design — stop_runs, then no-op until the run reaches a finished status on a later iteration, then delete_runs (or rely on submit_run's name-collision branch at runs/__init__.py:716, which itself calls delete_runs and has the same active-run constraint). The 8.3 stop_service agent tool ('runs.stop_runs(abort=True) + delete_runs') needs the same wait-until-finished handling. + +## [minor] [PARTIALLY_WRONG] 6.2 _process_to_be_deleted_endpoint +CLAIM: Stop and delete the backing run if present (stop_runs(abort=False), tolerate ResourceNotExistsError; then delete_runs once finished ...) +CORRECTION: The two-iteration stop-then-delete-once-finished flow is correct, but 'tolerate ResourceNotExistsError' targets the wrong exception: neither stop_runs nor delete_runs raises ResourceNotExistsError for missing run names — both just filter RunModel.run_name.in_(names) and silently no-op on zero matches (services/runs/__init__.py:873-880, 915-921). The error to guard against is ServerClientError from delete_runs when the run is not yet finished (runs/__init__.py:932-936). + +## [minor] [PARTIALLY_WRONG] 6.4 Direct model probe (health check) +CLAIM: Replicate _execute_probe (scheduled_tasks/probes.py:106-126) as check_model_endpoint(...) -> bool [snippet returns resp.is_success with no exception handling] +CORRECTION: The real _execute_probe wraps the whole tunnel+request in try/except (SSHError, httpx.RequestError) and returns False on failure (probes.py:112-126); it also passes follow_redirects=False. The plan's snippet omits the except clause, so an unreachable/dead replica raises SSHError (from get_service_replica_tunnel) or httpx.ConnectError instead of returning False; the Worker's generic exception handler (base.py:360-361) would then skip the endpoint's final update entirely — the health_check_failures counting path in 6.2 would never execute for exactly the failure mode it is meant to count. check_model_endpoint must catch (SSHError, httpx.RequestError) → False. Everything else in the snippet is accurate: get_service_replica_client(job: JobModel) is an async context manager (services/jobs/job_replica_http_client.py:22), the dummy 'http://dstack' host matches probes.py:116, the body/headers match _openai_model_probe_spec (services/jobs/configurators/base.py:472-493), orjson is a base dependency (pyproject.toml [project].dependencies), and the tunnel targets job_spec.service_port directly (job_replica_tunnel.py:25-43), so it is gateway-independent and needs no HTTP auth. + +## [minor] [PARTIALLY_WRONG] 2.3 Multi-replica safety (heartbeat semantics) +CLAIM: a Heartbeater extending lock_expires_at every ~1s while a worker processes an item +CORRECTION: The heartbeat loop runs every ~1s (heartbeat_delay=1.0, base.py:172,189), but it only writes lock_expires_at when the lease is within the heartbeat_trigger margin of expiry (base.py:213: item.lock_expires_at < now + self._hearbeat_margin). With lock_timeout=30s and heartbeat_trigger=15s each item's lease is extended roughly every ~15s, not every ~1s. The load-bearing part of the claim — indefinite extension for minutes-long process() calls, with untrack-on-missed-heartbeat and token-guarded writes as the backstop — is correct (base.py:204-254; precedent: instances/cloud_provisioning.py::create_cloud_instance offer loop). + +# Verifier 2 + +OVERALL: The plan's fetch/lock/cadence mechanics and its reading of submit_run's name-collision semantics are accurate, but the state machine has three blocker-level logic failures around retry and crash recovery: (1) the SUBMITTED handler has no reconcile step, so the advertised "run created but FK write lost" recovery actually ends in a terminal FAILED endpoint plus an orphaned, unstoppable service run; (2) the retry-after-run-failure branch never deletes the failed run, whose name keeps matching the reconcile fallback, making the "no run yet" retry branches unreachable for both preset and agent; (3) even fixed, PROVISIONING contains no preset re-submission path and no transition back to SUBMITTED, so preset retries no-op until the deadline. Two majors compound this: name-based reconcile/cleanup can adopt or destroy an unrelated pre-existing user run with the same name (the section 6.5 "submission fails cleanly" claim holds only for presets), and every "stop_runs + delete_runs" one-pass sequence in the plan always raises because stop is asynchronous and TERMINATING is not a finished status. All fixes are localized: reconcile-first in SUBMITTED with an ownership guard, delete the consumed run on retry, retry via status=SUBMITTED, and two-phase stop-then-delete everywhere. + +## [blocker] [WRONG] 6.1 / 6.3 (crash recovery of SUBMITTED preset submission) +CLAIM: "Run created but endpoint-row update lost the lock" is reconciled on the next iteration via run_id/run-name lookup (section 6.5). +CORRECTION: The reconcile-by-name lives only in _process_provisioning_endpoint (6.2 step 2). If the SUBMITTED handler's single token-guarded update {status: PROVISIONING, run_id, ...} is lost, the row is re-fetched still in SUBMITTED, and _process_submitted_endpoint (steps 1-4) has NO reconcile step — it re-runs preset matching and calls submit_run again with the same run name. submit_run (src/dstack/_internal/server/services/runs/__init__.py:716-718) unconditionally calls delete_runs on the colliding name, and delete_runs (:932-936) raises ServerClientError('Cannot delete active runs: ...') because the just-submitted run is SUBMITTED/PROVISIONING (not in RunStatus.finished_statuses(), core/models/runs.py:663). Per plan step 2 ('catch ServerClientError => FAILED'), the endpoint goes terminal FAILED while the orphaned service run keeps provisioning/running — and since run_id was never written and FAILED is terminal, neither the pipeline nor _process_to_be_deleted_endpoint ('backing run if present') will ever stop it. Minimal fix: make _process_submitted_endpoint reconcile-first — before matching/submitting, get_run_model_by_name(endpoint.name); if a non-deleted run exists that the endpoint plausibly owns (run.user_id == endpoint.user_id and run submitted_at >= endpoint.created_at), adopt it: write {status: PROVISIONING, run_id: run.id, provisioning_started_at: NOW, attempts: 1} and return. + +## [blocker] [WRONG] 6.2 step 2b (retry after backing-run failure) +CLAIM: Run in finished_statuses and attempts < MAX: clear run_id, stay PROVISIONING (next iteration re-runs preset/agent path). +CORRECTION: The failed run is never deleted, so it keeps existing non-deleted under the endpoint's name, and step 2's own fallback ('or a non-deleted run named after the endpoint exists') re-adopts it on EVERY subsequent iteration — get_run_model_by_name (runs/__init__.py:477-493) filters only deleted==False, so a FAILED run matches forever. The 'no run yet' branches (steps 3 and 4) are therefore unreachable after a failure: the handler loops finished-run -> clear run_id -> re-find-by-name until attempts hit MAX (if the state diagram's attempts++ applies per iteration) or the 1h deadline fires. Retry never actually re-provisions anything, for both preset and agent methods. Minimal fix: in the retry branch, soft-delete the consumed run (delete_runs in a fresh get_session_ctx — it is finished, so this succeeds) in the same iteration as clearing run_id, so the next iteration genuinely classifies as 'no run yet'. + +## [blocker] [WRONG] 6.2 step 4 (no run + method=preset) +CLAIM: Reaching here means the run vanished -> same retry logic as 2b (i.e., the preset path is re-run). +CORRECTION: There is no preset re-submission path reachable from PROVISIONING: preset submission exists only in _process_submitted_endpoint, step 3 of the PROVISIONING handler is explicitly gated on method=agent, and no transition ever returns the endpoint to SUBMITTED. So even with finding #2 fixed, a preset endpoint whose run failed sits in PROVISIONING as a pure no-op loop (10s cadence) until the 1h deadline, then FAILED 'Provisioning timed out' — the attempts=2 retry budget is dead code for presets. The plan also never defines whether a failed preset attempt may fall through to the agent (provisioning_method stays 'preset:', so as written it cannot). Minimal fix: on retry, transition back to {status: SUBMITTED, run_id: NULL, provisioning_method: NULL} (keeping attempts and provisioning_started_at) so the SUBMITTED dispatcher re-decides preset-vs-agent; state explicitly whether re-matching may pick the agent on attempt 2. + +## [major] [PARTIALLY_WRONG] 6.5 consequence (a) / 6.2 steps 2-3 (name-based reconcile vs pre-existing user run) +CLAIM: An existing active user run with the same name makes submission fail -> endpoint FAILED with a clear message. +CORRECTION: True only for the preset path (submit_run in SUBMITTED raises via delete_runs, runs/__init__.py:932-936). On the agent path no submission happens in SUBMITTED, so the endpoint enters PROVISIONING, where step 2's name-fallback ('or a non-deleted run named after the endpoint exists') adopts the user's unrelated pre-existing run as the backing run — if it is RUNNING+registered and probes OK, the endpoint goes ACTIVE bound to a run it does not own; worse, step 3's pre-launch cleanup ('if a leftover run named after the endpoint exists ... stop_runs(abort=True) + delete_runs') will abort and destroy the user's run. Nothing in create_endpoint (5.1) checks run-name collisions, so this is trivially reachable. Minimal fix: guard every name-based reconcile/cleanup with an ownership check (run.user_id == endpoint.user_id AND run submitted_at >= endpoint.created_at, or record the submitted run id in an interim token-guarded update before submit_run); alternatively reject/FAIL in SUBMITTED when a non-deleted foreign run already holds the name. + +## [major] [WRONG] 6.2 step 3 pre-launch cleanup / 8.3 stop_service tool +CLAIM: Terminate the leftover run first (stop_runs(abort=True) + delete_runs) so the agent starts clean. +CORRECTION: stop_runs is asynchronous: it only sets RunStatus.TERMINATING and defers to RunPipeline (runs/__init__.py:899-904, 'The run will be terminated by RunPipeline'). TERMINATING is not in finished_statuses() (core/models/runs.py:663-664), so an immediately-following delete_runs raises ServerClientError('Cannot delete active runs') every time — the one-pass 'stop + delete' sequence as written always fails, both in the pipeline pre-launch cleanup and in the agent's stop_service tool (8.3). Minimal fix: make it two-phase like _process_to_be_deleted_endpoint — issue stop_runs(abort=True), then treat 'leftover run still TERMINATING' as a no-op wait state (return, next iteration retries); only call delete_runs once the run reaches a finished status. Same fix applies inside the stop_service tool (poll with a short timeout before delete, or make delete best-effort). + +## [minor] [PARTIALLY_WRONG] 6.2 _process_to_be_deleted_endpoint (attack 5) +CLAIM: Deletion may take two iterations: first stop, then when the run reaches a finished status, delete it and mark the endpoint deleted. +CORRECTION: There is no bound on the wait: stop_runs(abort=False) is a graceful stop (TERMINATING set at runs/__init__.py:899-901, actual termination delegated to RunPipeline), and if the run is stuck TERMINATING the endpoint stays to_be_deleted forever, re-processed every 10s; the CLI's delete_configuration 'poll until gone' (5.3) never returns and the endpoint name is never freed (uniqueness is over non-deleted rows, section 4). Minimal fix: add a deletion deadline (e.g. reuse ENDPOINT_PROVISIONING_TIMEOUT or a fixed 10-15 min) after which the handler escalates to stop_runs(abort=True), and after a further grace period marks the endpoint deleted anyway with a status_message noting the run was left TERMINATING. + +## [minor] [PARTIALLY_WRONG] 4 / 6.5 (run_id FK ondelete=SET NULL, attack 7) +CLAIM: run_id: FK('runs.id', ondelete='SET NULL') — the authoritative link; reconcile handles the run going away. +CORRECTION: SET NULL is effectively dead: runs are only ever soft-deleted (delete_runs sets RunModel.deleted=True, runs/__init__.py:937-939; no server code hard-deletes RunModel rows — the only hard delete is the projects.id CASCADE, which also cascades the endpoint row itself). So after a user deletes the backing run, endpoint.run_id still resolves to a row with deleted=True, and the plan's PROVISIONING step 2 ('if run_id set ... load it') specifies no RunModel.deleted check — only the ACTIVE handler (step 1) mentions 'deleted'. It happens to behave acceptably (soft-deleted runs are always finished, so the retry branch fires), but the retry then interacts with finding #2 (the name-fallback correctly excludes deleted runs, so this path uniquely CAN reach 'no run yet'). Minimal fix: state explicitly that loading by run_id must treat run.deleted==True as 'run gone' in both PROVISIONING and ACTIVE handlers, and drop or annotate the SET NULL as project-cascade-only. + +# Verifier 3 + +OVERALL: Verified claude-agent-sdk 0.2.110 empirically in an isolated venv. The plan's §8 spike-hedged facts are mostly right (options fields, in-process MCP tools, bundled self-contained CLI binary, no node requirement, async IO), but three corrections matter: (1) BLOCKER — the `agent = ["claude-agent-sdk>=X"]` extra is uninstallable with dstack's `pydantic>=1.10.10,<2.0.0` pin (pyproject.toml:29) because mcp requires pydantic>=2.11; pip ResolutionImpossible verified, so the §11-Q4 plain-`anthropic` fallback (pydantic>=1.9,<3, compatible) becomes the mandatory v1 path unless dstack migrates to pydantic v2. (2) `ClaudeAgentOptions.env` merges over the full inherited server environment — there is no scrub mechanism; the §8.5 env-hygiene guarantee requires a custom Transport. (3) `allowed_tools` is a permission auto-approve list, not a toolset restriction — the hard no-Bash/no-filesystem boundary is `tools=[]` plus `allowed_tools` for the MCP tool names. + +## [blocker] [WRONG] 8.2 (packaging) / 9 / 11-Q4 +CLAIM: Packaging: new optional extra in pyproject.toml — agent = ["claude-agent-sdk>=X.Y"] (depends on dstack[server]), included in all +CORRECTION: The extra is not installable alongside dstack as pinned today. dstack pins `pydantic>=1.10.10,<2.0.0` (pyproject.toml:29), while claude-agent-sdk 0.2.110 requires `mcp>=1.23.0,<2.0.0` and mcp (1.28.1) requires `pydantic>=2.11.0,<3.0.0`. Verified empirically: `pip install --dry-run 'claude-agent-sdk>=0.2' 'pydantic>=1.10.10,<2.0.0'` fails with ResolutionImpossible. mcp has been a hard dependency since 0.1.0 and is required for the plan's in-process tools (`create_sdk_mcp_server` returns `McpSdkServerConfig`, `tool()` uses `mcp.types.ToolAnnotations`), so no SDK version avoids it. The §11 Q4 escape hatch — a plain `anthropic` tool-use loop — IS compatible (anthropic 0.116.0 requires `pydantic>=1.9.0,<3`), so it must be promoted from 'fallback' to the v1 default unless dstack migrates to pydantic v2 first. Also note mcp drags in pydantic-settings, starlette, uvicorn, jsonschema, pyjwt — a large transitive surface into the server env. + +## [major] [PARTIALLY_WRONG] 8.5 (env hygiene) +CLAIM: Env hygiene: the agent subprocess/client receives only ANTHROPIC_API_KEY (+ whatever the SDK minimally needs) — never the server's environment (verify the SDK's env-passing mechanism) +CORRECTION: `ClaudeAgentOptions.env` does NOT scrub — it merges OVER the full inherited server environment. Verified in claude_agent_sdk/_internal/transport/subprocess_cli.py::SubprocessCLITransport.connect (v0.2.110, lines 430-436): `inherited_env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}` then `process_env = {**inherited_env, "CLAUDE_CODE_ENTRYPOINT": "sdk-py", **self._options.env, ...}`. There is no option to start from a clean env; by default the CLI subprocess inherits DB credentials, cloud creds, encryption keys — the exact blast radius §8.3 tries to eliminate. Achieving the scrub requires a custom transport: `query()` and `ClaudeSDKClient` both accept `transport:` and `Transport` is a public ABC (connect/write/read_messages/close/is_ready/end_input), or subclass the internal `SubprocessCLITransport` and rebuild `process_env`. ANTHROPIC_API_KEY itself is consumed by the CLI subprocess from its env (pass via options.env); the Python SDK reads it only in _internal/session_resume.py. + +## [major] [PARTIALLY_WRONG] 8.2 / 8.3 (tool allowlist) +CLAIM: permissioning (allowed_tools) gives a hard tool allowlist; the agent gets only custom in-process dstack tools — no Bash, no filesystem tools; allowed_tools = exactly these mcp__dstack__* names +CORRECTION: `allowed_tools` maps to `--allowedTools` (subprocess_cli.py::_build_command), which is a permission auto-approve list — it does not remove built-in tools from the toolset. The hard toolset restriction is the separate `tools` field (`ClaudeAgentOptions.tools: list[str] | ToolsPreset | None`), which maps to `--tools`; `tools=[]` emits `--tools ""` giving an EMPTY base toolset (no Bash/Read/Write/WebFetch). The correct hard boundary is `tools=[]` (removes all built-ins) combined with `allowed_tools=["mcp__dstack__*" names]` (auto-approves the MCP tools so headless runs don't hit permission denials); optionally `disallowed_tools` as belt-and-suspenders. Also relevant: `setting_sources` defaults to None = no filesystem settings loaded, but passing `skills=` silently defaults setting_sources to ['user','project'] and appends 'Skill' to allowed_tools — do not use `skills` in the server context. + +# Verifier 4 + +OVERALL: Section 7's matching mechanism is factually sound: the _get_job_plan REUSE behavior, CreationPolicy literals, IDLE→is_available() semantics, str→OpenAIChatModel coercion, Env.update, the ProfileParams merge-loop precedent, Optional RunSpec.profile, and get_plan's commit-free/repo-free/plugin-safe behavior all check out at the cited symbols. The one correction that will break at runtime as written: the endpoint worker's refetch must eager-load ProjectModel.backends (volumes.py:237 pattern), because get_plan lazy-accesses project.backends and async SQLAlchemy raises MissingGreenlet otherwise. Secondary corrections: get_plan raises ServerClientError at match time (SSH key, invalid merged config) so find_matching needs per-candidate error handling, REUSE planning still enumerates cloud backend offers per candidate fleet (latency/side-effects in the background loop), and the ≥1-available-offer criterion under-checks capacity for multi-replica presets. + +## [major] [PARTIALLY_WRONG] §6.1 worker refetch + §7.3 matching via get_plan +CLAIM: The worker refetches the endpoint row 'with joinedloads (project, user, run)' and can then call find_matching→get_plan on its session +CORRECTION: get_plan reaches backends_services.get_project_backends(project), which iterates the lazy `project.backends` relationship (services/backends/__init__.py:281 in get_project_backends_with_models) — under async SQLAlchemy this raises MissingGreenlet unless eager-loaded. Every existing pipeline that reaches backend code chains the load, e.g. volumes pipeline: `.options(joinedload(VolumeModel.project).joinedload(ProjectModel.backends))` (background/pipeline_tasks/volumes.py:237); same pattern in gateways.py:232, jobs_submitted.py:821. The plan's joinedload list must include `EndpointModel.project → ProjectModel.backends` (or find_matching must load the project with backends in its own session). This bites whenever any candidate fleet is a cloud fleet: find_optimal_fleet_with_offers → _get_backend_offers_in_fleet → get_offers_by_requirements → get_project_backends (offers.py:44); only pure-SSH-fleet projects escape via check_can_create_new_cloud_instance_in_fleet raising ValueError (plan.py:716-723). + +## [minor] [PARTIALLY_WRONG] §7.3 — matching error handling / SSH key requirement +CLAIM: SSH-key requirement and ServerClientError handling only matter at submission (submit_run / register_service); matching just calls get_plan per candidate +CORRECTION: get_plan itself calls validate_run_spec_and_set_defaults (runs/__init__.py:544-548), which raises ServerClientError("ssh_key_pub must be set if the user has no ssh_public_key") at runs/spec.py:128-132 — so the SSH-key requirement bites at MATCH time, not just submission (mitigated: create_user always generates a keypair, services/users.py:176-186, but e.g. legacy/SSO-created users without keys would fail matching, and validate can also raise for a bad merged preset: probe caps runs/spec.py:107-117, volume mount paths :79-81, scheduled+scale-to-zero :100-106). find_matching must catch ServerClientError per candidate and treat it as no-match/skip, otherwise one bad preset aborts matching; the plan only specifies catching ServerClientError around submit_run (§6.2 step 2) and 'invalid files skipped' at YAML-parse level (§7.2). + +## [minor] [PARTIALLY_WRONG] §2.7 / §7.3 — cost of REUSE matching +CLAIM: get_plan with creation_policy=reuse is effectively an existing-fleet-only check +CORRECTION: The RETURNED offers are existing-instance-only (correct), but the planner still enumerates cloud backend offers for every candidate cloud fleet regardless of creation_policy: get_job_plans calls find_optimal_fleet_with_offers without skip_backend_offers_on_pool_capacity (plan.py:171-180), so `skip_backend_offers` is False (plan.py:405) and _get_backend_offers_in_fleet → get_offers_by_requirements runs per fleet (plan.py:416-424, offers.py:30+, hitting backend Compute.get_offers), then again uncapped for the optimal fleet (plan.py:449-456); the results are discarded by _get_job_plan under REUSE. Consequence for §7.3: each preset candidate × replica group evaluation in the background worker performs cloud-offer enumeration (network/catalog calls, seconds each) — harmless for correctness (heartbeater covers it) but not the light check the plan implies; worth noting or bounding candidate count. + +## [minor] [PARTIALLY_WRONG] §7.3 / §11.6 — match criterion for multi-replica presets +CLAIM: Match ⇔ every job_plan has ≥1 offer with offer.availability.is_available() +CORRECTION: For presets with replica groups where count.min > 1 this criterion under-checks capacity: get_job_plans plans replica_num=0 per replica group (plan.py:136-142), so one available instance satisfies the criterion even though get_nodes_required_num = sum of count.min (runs/spec.py:250-258) instances are needed. When no fleet has full pool capacity (has_pool_capacity requires nodes_required_num ≤ available instance offers, plan.py:393-399), the optimal fleet is chosen by price and can still contribute one available offer → 'match' for a service that cannot fully place on existing fleets. Acceptable given the plan calls matching advisory (fleet-drift note), but the criterion is 'each job can start', not 'the whole service fits'; either accept explicitly or check total available offers ≥ nodes_required_num. + +# Verifier 5 + +OVERALL: This is an unusually implementable plan: every code-level ground-truth claim I spot-checked against master@28ea5f86f confirmed exactly (submit_run/get_plan signatures, _get_job_plan's reuse-only offer behavior, pipeline_tasks/base.py symbols, PipelineModelMixin, Target.from_model, Env.update, RunModel.service_spec, the volumes lock-retry delete, the dual-core-model config pattern). The blocking problems are lifecycle-semantics holes concentrated in §6.2 — the preset retry path is circular and never re-submits, and FAILED transitions are inconsistent about backing-run teardown — plus two cross-section conflicts: the volumes-style delete API cannot acquire the lock during the minutes-long lease-held agent step, and the health probe's model_name/prefix inputs are unspecified with the obvious guess (endpoint config's model string) being wrong. Fixing the eight findings above, most of which are one-paragraph edits to §6.2/§8.3, would make M1-M5 implementable without re-research. + +## [blocker] [WRONG] §6.2 _process_provisioning_endpoint (steps 2 & 4) + state diagram +CLAIM: On backing-run failure with attempts < MAX: "clear run_id, stay PROVISIONING (next iteration re-runs preset/agent path)"; step 4 (no run + method=preset) says "same retry logic as 2b". +CORRECTION: This is circular: no step ever re-submits a preset run from PROVISIONING. Step 3 only handles method=agent; step 4 points back to 2b, which only clears run_id — so a preset endpoint whose run failed spins as a no-op until the 1h deadline. Also, the state diagram says "attempts++" on retry but the prose never says where attempts is incremented for the preset path (SUBMITTED sets attempts=1 only on first submission). Fix: add an explicit "no run yet + method=preset:*" step that increments attempts (token-guarded, like the agent path) and re-runs EndpointPresetService.find_matching + merged submit_run in a fresh session — and state whether re-matching may switch the method to agent when the preset no longer matches (recommend: yes, re-dispatch exactly as _process_submitted_endpoint does). Alternative: transition back to SUBMITTED and let _process_submitted_endpoint re-dispatch. Pick one and write it into §6.2. + +## [major] [WRONG] §5.1 delete_endpoints vs §6.2 step 3 / §8.5 +CLAIM: delete_endpoints uses the volumes-style lock-retry (10×, ServerClientError if rows are pipeline-locked), while the agent step legitimately holds the heartbeated lease for minutes up to the 1h deadline. +CORRECTION: These two settled decisions contradict each other: the volumes pattern (services/volumes.py::delete_volumes — retries 10×0.5s for lock_expires_at IS NULL, then raises "volumes are being processed currently") assumes short processing steps, but §6.2's in-worker agent run keeps lock_expires_at perpetually extended, so `dstack delete` on a PROVISIONING(agent) endpoint deterministically fails for the entire agent run. Fix: for endpoints, set to_be_deleted=True with a plain UPDATE that does not require acquiring the pipeline lock (safe — the flag is only consumed by the pipeline; the worker's final token-guarded write should re-check it), and have the agent step periodically check to_be_deleted and abort. If instead you keep the volumes pattern, the plan must say deletion is rejected during agent provisioning and the CLI must surface that. + +## [major] [PARTIALLY_WRONG] §6.4 Direct model probe +CLAIM: check_model_endpoint(job_model, model_name, prefix="/v1") — the plan never says where model_name and prefix come from. +CORRECTION: The mechanism (get_service_replica_client + dummy host) is verified correct, but the inputs are undefined and the obvious guess — endpoint configuration's `model` string — is wrong: §7.3 matches presets case-insensitively and the agent may serve under a different name (e.g. --served-model-name), while chat/completions model routing is name-sensitive; prefix is also not always /v1. Specify: model_name = the backing run's ServiceSpec.model.name (RunModel.service_spec, server/models.py:446; ServiceModelSpec, core/models/runs.py:635 — note it has name/base_url/type, no prefix field) and prefix = the backing service configuration's model.prefix (OpenAIChatModel.prefix, core/models/services.py:72). + +## [major] [PARTIALLY_WRONG] §8.3 submit_service tool / §1 deliverable +CLAIM: The endpoint's deliverable URL comes from ServiceSpec.model.base_url and activation requires a model probe, but nothing enforces that the agent-submitted service YAML declares `model:` — only prompt guidance in §8.4. +CORRECTION: An agent deployment without `model:` can reach RUNNING + registered, leaving ServiceSpec.model = None — Endpoint.url stays None and the health probe has no model name, so activation semantics are undefined. Add a hard check to the submit_service tool: reject (or auto-inject with the endpoint's model value) service YAML whose `model:` is unset. Note it in the §8.3 table and add it to M4's tests. (Presets are already safe: §7.3 matching requires configuration.model.name.) + +## [major] [PARTIALLY_WRONG] §6.2 ACTIVE→FAILED and agent-exhausted FAILED transitions +CLAIM: Only the deadline-exceeded branch says "stop the backing run if any"; the health-check FAILED ("Model endpoint stopped responding") and the final agent-failure FAILED say nothing about the backing run, and FAILED rows are terminal (excluded from the fetcher). +CORRECTION: In both unstated cases a GPU-consuming service run can still be RUNNING and will leak indefinitely, since FAILED endpoints are never re-fetched and agent cleanup only happens before the *next* attempt. State explicitly, per FAILED transition, whether the backing run is stopped. Recommended: stop_runs on every FAILED transition, matching the deadline branch. If deliberately left running (e.g. for debugging), document that choice and that deleting the FAILED endpoint still stops+deletes the run via _process_to_be_deleted_endpoint. + +## [minor] [PARTIALLY_WRONG] §3 EndpointSpec vs §5.2 CreateEndpointRequest +CLAIM: EndpointSpec{configuration, configuration_path} is defined in core/models/endpoints.py, but CreateEndpointRequest takes bare `configuration: EndpointConfiguration` and nothing in v1 consumes EndpointSpec (only §12's Later plugins item references it). +CORRECTION: A dead type invites the implementer to guess whether configuration_path should be wired through create/CLI. Either drop EndpointSpec from the v1 file (introduce it with the Later plugins work), or change CreateEndpointRequest to take `spec: EndpointSpec` and have the CLI configurator populate configuration_path (RunSpec precedent). State which. + +## [minor] [PARTIALLY_WRONG] §6.2 _process_provisioning_endpoint step 2 +CLAIM: "probe OK ⇒ ACTIVE (reset health_check_failures), record URL source" +CORRECTION: "record URL source" is an undefined operation: §4's EndpointModel has no url column and §6.1's _EndpointUpdateMap has no url field, while §5.1 says url is derived at read time from the joined run's service_spec. Either delete the phrase (nothing needs recording if url is always read-time-derived) or, if something must be persisted, add the column to §4 and the field to the update map. + +## [minor] [PARTIALLY_WRONG] §10 M2 vs §9 settings +CLAIM: M2 implements "deadline/attempts" and ACTIVE monitoring, but no milestone includes adding DSTACK_SERVER_ENDPOINT_PROVISIONING_TIMEOUT or the code-level constants (ENDPOINT_PROVISIONING_MAX_ATTEMPTS, ENDPOINT_ACTIVE_CHECK_INTERVAL, ENDPOINT_HEALTH_CHECK_MAX_FAILURES) to server/settings.py — M3 covers only ENDPOINT_PRESETS_DIR and M4 only the agent settings. +CORRECTION: Add to the M2 bullet: "settings.py: ENDPOINT_PROVISIONING_TIMEOUT env var (+ env.md entry per the module docstring rule) and the three code-level constants". While there, reconcile §9 calling ENDPOINT_PROVISIONING_MAX_ATTEMPTS a "code-level constant (not env)" with §6.2 referencing it as settings.ENDPOINT_PROVISIONING_MAX_ATTEMPTS — say they live as plain module constants in settings.py. diff --git a/pyproject.toml b/pyproject.toml index ca08438931..1081770403 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,11 +62,17 @@ artifacts = [ "src/dstack/_internal/server/statics/**", ] +[tool.hatch.build.targets.sdist.force-include] +"skills" = "skills" + [tool.hatch.build.targets.wheel] artifacts = [ "src/dstack/_internal/server/statics/**", ] +[tool.hatch.build.targets.wheel.force-include] +"skills" = "skills" + [tool.hatch.metadata.hooks.fancy-pypi-readme] content-type = "text/markdown" diff --git a/skills/dstack-prototyping/SKILL.md b/skills/dstack-prototyping/SKILL.md new file mode 100644 index 0000000000..a51e245f88 --- /dev/null +++ b/skills/dstack-prototyping/SKILL.md @@ -0,0 +1,241 @@ +--- +name: dstack-prototyping +description: | + Use with the dstack skill when a dstack workload, especially model serving, needs research, hardware sizing, recipe selection, live experiments, or debugging before it can be considered ready. Guides how to use dstack dev environments, tasks, services, offers, logs, events, and run JSON as an evidence loop without duplicating dstack CLI/YAML syntax. +--- + +# dstack Prototyping + +## Required Companion + +Load `/dstack` first. `/dstack` is the authority for CLI syntax, YAML fields, confirmation rules, attach behavior, logs, events, offers, fleets, services, and command safety. + +This skill decides what to investigate, what evidence to collect, when to use each dstack run type, and when a workload is proven. Do not repeat or override `/dstack` command syntax here. If a command or field is uncertain, follow `/dstack`: check help/docs before using it. + +## Operating Standard + +Treat prototyping as an evidence loop: + +1. define the target behavior; +2. build a model/workload/hardware dossier; +3. choose the smallest experiment that can remove the largest uncertainty; +4. run through dstack using `/dstack` rules; +5. classify the result; +6. change only the next unproven assumption; +7. promote only after functional verification. + +Do not chase a local optimum. Step back after each failed candidate and ask whether the problem is the recipe, hardware, image, dstack placement, backend provisioning, auth, model format, or verification method. + +## Source Stack + +Prefer sources that can directly change a deployment decision: + +1. model card, `config.json`, tokenizer/config files, official repo discussions, and model-family docs; +2. current serving recipe sources: + - `https://recipes.vllm.ai/` + - `https://recipes.vllm.ai/models.json` + - `https://docs.vllm.ai/projects/recipes/en/stable/` + - `https://lmsysorg.mintlify.app/cookbook/intro` + - `https://docs.sglang.ai/` +3. framework docs, release notes, issues, and image tags for vLLM, SGLang, TensorRT-LLM, TGI, or a model-specific server; +4. dstack evidence from offers, plans, run JSON, events, logs, and attached shells; +5. recent deployment writeups when they explain a decision pattern, not just marketing: + - `https://modal.com/blog/introducing-auto-endpoints` + - `https://www.runpod.io/blog/overdrive-benchmarks` + - `https://www.makora.com/` + - `https://www.wafer.ai/blog/glm52-amd` + - `https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development` + +Use competitor and research links to sharpen the loop: inspectable recipes, workload-specific tuning, engine-level metrics, framework patches, hardware-specific friction, and artifact contracts. Do not treat them as instructions to benchmark, optimize under load, or implement kernels unless the user asked for that scope. + +## Dossier Before Spending GPU + +Write a compact note before the first paid run: + +- requested model and exact served model name; +- target API shape: OpenAI chat, completion, embeddings, multimodal, custom; +- required proof: the smallest request that proves the requested behavior; +- model facts: architecture, parameter count, active parameters for MoE, quantization, context length, tokenizer quirks, license/gating; +- serving candidates and why: vLLM, SGLang, framework recommended by model card, or fallback; +- memory envelope: weight memory, KV cache pressure, tensor/pipeline/data parallel needs, disk cache, CPU/RAM expectations; +- dstack constraints inherited from the request: backends, fleets, spot policy, max price, env names, volumes, regions, or instance types; +- unknowns that require a dev environment, task, or service attempt. + +If this note cannot name the top two uncertainties, do more source inspection before submitting work. + +## Hardware Envelope + +Derive requirements before looking at offers. Offers answer placement, not correctness. + +Use broad scheduling requirements until a real failure justifies narrowing: + +- express GPU need as memory/count first; +- avoid pinning GPU model, backend, region, CPU, RAM, disk, or instance type from the first acceptable offer; +- require exact GPU model only for a known kernel/runtime path, unsupported architecture, benchmark target, or reproduced failure; +- check serving image/runtime compatibility with the backend driver before trusting the first run; +- set disk from model cache plus image/runtime overhead, not from a random offer; +- keep price and spot/on-demand constraints from the user/profile. + +Separate three hardware concepts: + +- **minimum requirement**: what the service config should request for scheduling; +- **candidate offer**: what dstack may place now; +- **tested hardware**: what actually ran and passed verification. + +Never collapse tested hardware back into minimum requirements without evidence. + +## Framework Choice + +Start with vLLM and SGLang for OpenAI-compatible LLM serving unless the model card points elsewhere. + +Prefer vLLM when: + +- vLLM recipes or docs provide a current command for the model/hardware; +- the model is standard dense/decoder-only and the goal is a straightforward OpenAI-compatible service; +- the recipe gives usable tensor-parallel, dtype, quantization, or image guidance. + +Prefer SGLang when: + +- SGLang Cookbook has a model-specific recipe; +- the model path needs SGLang-specific support, speculative decoding, radix/KV reuse, structured output performance, multimodal support, or long-context behavior; +- vLLM support is absent, degraded, or known to miss a required quantization/model path. + +Consider another server only after evidence says both vLLM and SGLang are poor fits for the requested model or API. + +For advanced serving shapes, keep scope honest. P/D disaggregation, multi-service router/worker layouts, autoscaling tuning, benchmarking loops, speculative decoding tuning, kernel changes, and production load optimization are separate work unless required to make the model serve at all. + +## Choosing The Experiment + +Submit a service directly only when these are already known: + +- pinned image or install path; +- launch command and port; +- model/API shape; +- resource envelope; +- expected health and final verification request. + +Do not use `:latest` for a final serving image unless a source or a prior run proves it is compatible with the selected backend/runtime. If a run fails because CUDA/PyTorch/vLLM requires a newer NVIDIA driver than the host provides, change the image/runtime tag or select compatible hardware; do not retry the same image. + +Use a dev environment when an interactive shell can answer a main uncertainty faster than repeated service submissions: + +- GPU runtime/image compatibility; +- import/install friction; +- model download/auth; +- launch flags; +- memory at load time; +- framework-specific error messages; +- a command that must be tuned before it becomes a service. + +Use a task when the question is one-shot: + +- `nvidia-smi`/driver/runtime sanity; +- package import or version; +- model cache/download check; +- short server start/probe; +- backend provisioning smoke test. + +Use a service when URL wiring, probes, serving process lifetime, or final API behavior is the question. + +For container-style backends that cannot pre-provision reusable instances, do not invent a pre-provisioning step. Use plans/offers and detached runs. For VM or SSH fleets, dev environments can be efficient because image/runtime work can be reused interactively. + +## Candidate Discipline + +Every candidate must have a reason to exist. Record: + +- hypothesis; +- run type; +- run name, different from any higher-level object whose logs or status are being tracked; +- config path; +- framework/image/command; +- requested resources; +- dstack plan result; +- run name/id after submission; +- current status; +- decision: keep, promote, retry with change, or stop; +- exact reason for the decision. + +Change one important variable at a time unless the previous candidate failed before testing any useful assumption. Examples of valid next changes: + +- image tag or install method; +- vLLM vs SGLang; +- dtype/quantization; +- tensor parallelism; +- GPU memory/count requirement; +- model name/path or trust-remote-code style flag when documented; +- auth/env/volume/model-cache handling; +- backend/fleet constraint after a placement or provisioning diagnosis. + +Retrying the same YAML after the same error is not prototyping. + +For endpoint deployment agents, do not name a candidate service exactly like the endpoint. Endpoint logs and service logs must stay distinguishable during failure diagnosis. Prefer short attempt names and increment them only when submitting a materially different candidate. + +## Reading dstack Evidence + +Use `/dstack` for the exact commands. Interpret evidence this way: + +- plan output: placement and price preview only; +- offers: current capacity candidates only; +- run JSON: authoritative run identity, status, service URL/model fields, jobs, and actual placed resources; +- events: lifecycle and backend/provisioning transitions; +- logs: process behavior, image/model/framework errors; +- attached shell: only for interactive diagnosis or command tuning, not for final proof. + +When waiting for a service, poll run JSON and stop waiting on terminal states. Do not wait only for log markers such as "Uvicorn running"; the process may already have failed, and the next useful action is log diagnosis. + +Use normal logs before diagnostic logs. Diagnostic logs can contain runtime environment details; use them only when normal logs and run JSON are not enough to identify whether the issue is dstack/backend infrastructure rather than the workload. + +If dstack/backend behavior appears broken, isolate it with a minimal non-application reproduction outside the repo you are working in. Do not let endpoint-agent mistakes and backend bugs blur together. + +## Failure Routing + +Classify before editing: + +- no matching offer or no fleet; +- budget/profile constraint too narrow; +- backend provisioning stalled or failed; +- image pull/runtime mismatch; +- dependency import/install failed; +- model download/gated auth/cache failed; +- server process exited; +- OOM at load or during first request; +- port/probe/service URL mismatch; +- model API works but wrong model name/API shape; +- dstack bug or backend integration bug. + +For provisioning stalls, inspect dstack run JSON and events first. Use SSH/backend-native inspection only when dstack evidence points to an instance/backend problem and the user/project context permits it. + +## Final Service Proof + +Do not treat `running`, a passed service probe, or clean logs as final success. + +For OpenAI-compatible model serving, success requires a real request to the model endpoint: + +- use the requested model name; +- include authentication only as required by dstack/service config; +- verify HTTP success; +- verify response schema; +- verify generated content exists; +- verify no model-name mismatch; +- record the URL shape, request body without secrets, response evidence, run id/name, and actual hardware from run JSON. + +If the final deliverable is a service, dev environments and tasks are evidence only. Promote the working command into a clean service and verify that service. + +## Promotion + +Promote a candidate only when the service is clean and reproducible: + +- remove debug/install/probe commands that were only for investigation; +- include applicable backend, fleet, price, spot, env-name, volume, and region constraints in the final YAML instead of relying only on apply-time flags; +- preserve the working image, command, port, model, env names, volumes, and required resource envelope; +- preserve broad minimum resources separately from actual tested hardware; +- keep source URLs and important run evidence with the final report; +- stop or mark ruled-out paid candidates unless they are still needed for active debugging. + +The final report should let another agent or developer understand: + +- why this framework and hardware envelope were chosen; +- which alternatives failed or were rejected; +- which source URLs grounded the recipe; +- the final service run id/name; +- the exact verification request/result; +- the actual hardware that passed. diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py index 0dacbc1415..ef7cec4719 100644 --- a/src/dstack/_internal/cli/commands/endpoint.py +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -1,15 +1,22 @@ import argparse +import base64 +import sys import time +from typing import Iterable from rich.live import Live from dstack._internal.cli.commands import APIBaseCommand -from dstack._internal.cli.services.completion import EndpointNameCompleter +from dstack._internal.cli.services.completion import ( + EndpointNameCompleter, + EndpointPresetNameCompleter, +) from dstack._internal.cli.utils.common import ( LIVE_TABLE_PROVISION_INTERVAL_SECS, LIVE_TABLE_REFRESH_RATE_PER_SEC, confirm_ask, console, + get_start_time, ) from dstack._internal.cli.utils.endpoint import ( filter_endpoints_for_listing, @@ -17,7 +24,10 @@ print_endpoint, print_endpoints_table, ) +from dstack._internal.cli.utils.preset import print_endpoint_presets_table from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.core.models.endpoints import Endpoint +from dstack._internal.server.schemas.logs import PollLogsRequest from dstack._internal.utils.json_utils import pydantic_orjson_dumps_with_indent @@ -62,19 +72,38 @@ def _register(self): default=None, ) - delete_parser = subparsers.add_parser( - "delete", - help="Delete endpoints", + logs_parser = subparsers.add_parser( + "logs", + help="Show endpoint logs", + formatter_class=self._parser.formatter_class, + ) + logs_parser.add_argument( + "name", + help="The name of the endpoint", + ).completer = EndpointNameCompleter() # type: ignore[attr-defined] + logs_parser.add_argument( + "--since", + help=( + "Show only logs newer than the specified date." + " Can be a duration (e.g. 10s, 5m, 1d) or an RFC 3339 string (e.g. 2023-09-24T15:30:00Z)." + ), + type=str, + ) + logs_parser.set_defaults(subfunc=self._logs) + + stop_parser = subparsers.add_parser( + "stop", + help="Stop an endpoint", formatter_class=self._parser.formatter_class, ) - delete_parser.add_argument( + stop_parser.add_argument( "name", help="The name of the endpoint", ).completer = EndpointNameCompleter() # type: ignore[attr-defined] - delete_parser.add_argument( + stop_parser.add_argument( "-y", "--yes", help="Don't ask for confirmation", action="store_true" ) - delete_parser.set_defaults(subfunc=self._delete) + stop_parser.set_defaults(subfunc=self._stop) get_parser = subparsers.add_parser( "get", help="Get an endpoint", formatter_class=self._parser.formatter_class @@ -91,6 +120,51 @@ def _register(self): ) get_parser.set_defaults(subfunc=self._get) + preset_parser = subparsers.add_parser( + "preset", + help="Manage endpoint presets", + formatter_class=self._parser.formatter_class, + ) + preset_parser.set_defaults(subfunc=self._preset_list) + preset_subparsers = preset_parser.add_subparsers(dest="preset_action") + + preset_list_parser = preset_subparsers.add_parser( + "list", + help="List endpoint presets", + formatter_class=self._parser.formatter_class, + ) + preset_list_parser.set_defaults(subfunc=self._preset_list) + + preset_get_parser = preset_subparsers.add_parser( + "get", + help="Get an endpoint preset", + formatter_class=self._parser.formatter_class, + ) + preset_get_parser.add_argument( + "name", + help="The name of the endpoint preset", + ).completer = EndpointPresetNameCompleter() # type: ignore[attr-defined] + preset_get_parser.add_argument( + "--json", + action="store_true", + help="Output in JSON format", + ) + preset_get_parser.set_defaults(subfunc=self._preset_get) + + preset_delete_parser = preset_subparsers.add_parser( + "delete", + help="Delete endpoint presets", + formatter_class=self._parser.formatter_class, + ) + preset_delete_parser.add_argument( + "name", + help="The name of the endpoint preset", + ).completer = EndpointPresetNameCompleter() # type: ignore[attr-defined] + preset_delete_parser.add_argument( + "-y", "--yes", help="Don't ask for confirmation", action="store_true" + ) + preset_delete_parser.set_defaults(subfunc=self._preset_delete) + def _command(self, args: argparse.Namespace): super()._command(args) args.subfunc(args) @@ -118,21 +192,65 @@ def _get_endpoints_for_listing(self, args: argparse.Namespace): limit=args.last, ) - def _delete(self, args: argparse.Namespace): + def _logs(self, args: argparse.Namespace): + try: + endpoint = self.api.client.endpoints.get( + project_name=self.api.project, + name=args.name, + ) + except ResourceNotExistsError: + console.print("Endpoint not found") + exit(1) + + start_time = get_start_time(args.since) try: - self.api.client.endpoints.get(project_name=self.api.project, name=args.name) + for log in self._get_endpoint_logs(endpoint=endpoint, start_time=start_time): + sys.stdout.buffer.write(log) + sys.stdout.buffer.flush() + except KeyboardInterrupt: + pass + + def _get_endpoint_logs(self, endpoint: Endpoint, start_time) -> Iterable[bytes]: + next_token = None + while True: + resp = self.api.client.logs.poll( + project_name=self.api.project, + body=PollLogsRequest( + run_name=endpoint.name, + job_submission_id=endpoint.id, + start_time=start_time, + end_time=None, + descending=False, + limit=1000, + diagnose=False, + next_token=next_token, + ), + ) + for log in resp.logs: + yield base64.b64decode(log.message) + next_token = resp.next_token + if next_token is None: + break + + def _stop(self, args: argparse.Namespace): + try: + endpoint = self.api.client.endpoints.get(project_name=self.api.project, name=args.name) except ResourceNotExistsError: console.print(f"Endpoint [code]{args.name}[/] does not exist") exit(1) - if not args.yes and not confirm_ask(f"Delete the endpoint [code]{args.name}[/]?"): + if endpoint.status.is_finished(): + console.print(f"Endpoint [code]{args.name}[/] is already {endpoint.status.value}") + return + + if not args.yes and not confirm_ask(f"Stop the endpoint [code]{args.name}[/]?"): console.print("\nExiting...") return - with console.status("Deleting endpoint..."): - self.api.client.endpoints.delete(project_name=self.api.project, names=[args.name]) + with console.status("Stopping endpoint..."): + self.api.client.endpoints.stop(project_name=self.api.project, names=[args.name]) - console.print(f"Endpoint [code]{args.name}[/] deleted") + console.print(f"Endpoint [code]{args.name}[/] stopping") def _get(self, args: argparse.Namespace): try: @@ -148,3 +266,43 @@ def _get(self, args: argparse.Namespace): print(pydantic_orjson_dumps_with_indent(endpoint.dict(), default=None)) return print_endpoint(endpoint) + + def _preset_list(self, args: argparse.Namespace): + presets = self.api.client.endpoint_presets.list(self.api.project) + print_endpoint_presets_table(presets) + + def _preset_get(self, args: argparse.Namespace): + if not args.json: + console.print("Use --json to output the endpoint preset.") + exit(1) + try: + preset = self.api.client.endpoint_presets.get( + project_name=self.api.project, + name=args.name, + ) + except ResourceNotExistsError: + console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + exit(1) + print(pydantic_orjson_dumps_with_indent(preset.dict(), default=None)) + + def _preset_delete(self, args: argparse.Namespace): + presets = self.api.client.endpoint_presets.list(self.api.project) + if args.name not in {preset.name for preset in presets}: + console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + exit(1) + + if not args.yes and not confirm_ask(f"Delete the endpoint preset [code]{args.name}[/]?"): + console.print("\nExiting...") + return + + try: + with console.status("Deleting endpoint preset..."): + self.api.client.endpoint_presets.delete( + project_name=self.api.project, + names=[args.name], + ) + except ResourceNotExistsError: + console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + exit(1) + + console.print(f"Endpoint preset [code]{args.name}[/] deleted") diff --git a/src/dstack/_internal/cli/commands/logs.py b/src/dstack/_internal/cli/commands/logs.py index 3993050e73..575dfef8db 100644 --- a/src/dstack/_internal/cli/commands/logs.py +++ b/src/dstack/_internal/cli/commands/logs.py @@ -1,14 +1,11 @@ import argparse -import base64 import sys from typing import Iterable from dstack._internal.cli.commands import APIBaseCommand -from dstack._internal.cli.services.completion import RunOrEndpointNameCompleter +from dstack._internal.cli.services.completion import RunNameCompleter from dstack._internal.cli.utils.common import get_start_time -from dstack._internal.core.errors import CLIError, ResourceNotExistsError -from dstack._internal.core.models.endpoints import Endpoint -from dstack._internal.server.schemas.logs import PollLogsRequest +from dstack._internal.core.errors import CLIError from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -43,7 +40,7 @@ def _register(self): ), type=str, ) - self._parser.add_argument("run_name").completer = RunOrEndpointNameCompleter() # type: ignore[attr-defined] + self._parser.add_argument("run_name").completer = RunNameCompleter() # type: ignore[attr-defined] def _command(self, args: argparse.Namespace): super()._command(args) @@ -61,10 +58,6 @@ def _get_logs( args: argparse.Namespace, start_time, ) -> Iterable[bytes]: - endpoint = self._get_endpoint(args.run_name) - if endpoint is not None: - return self._get_endpoint_logs(endpoint=endpoint, args=args, start_time=start_time) - run = self.api.runs.get(args.run_name) if run is not None: return run.logs( @@ -74,51 +67,4 @@ def _get_logs( job_num=args.job, ) - raise CLIError(f"Run or endpoint {args.run_name} not found") - - def _get_endpoint(self, name: str) -> Endpoint | None: - try: - return self.api.client.endpoints.get( - project_name=self.api.project, - name=name, - ) - except ResourceNotExistsError: - return None - - def _get_endpoint_logs( - self, - endpoint: Endpoint, - args: argparse.Namespace, - start_time, - ) -> Iterable[bytes]: - if endpoint.run_name is not None: - run = self.api.runs.get(endpoint.run_name) - if run is not None: - yield from run.logs( - start_time=start_time, - diagnose=args.diagnose, - replica_num=args.replica, - job_num=args.job, - ) - return - - next_token = None - while True: - resp = self.api.client.logs.poll( - project_name=self.api.project, - body=PollLogsRequest( - run_name=endpoint.name, - job_submission_id=endpoint.id, - start_time=start_time, - end_time=None, - descending=False, - limit=1000, - diagnose=False, - next_token=next_token, - ), - ) - for log in resp.logs: - yield base64.b64decode(log.message) - next_token = resp.next_token - if next_token is None: - break + raise CLIError(f"Run {args.run_name} not found") diff --git a/src/dstack/_internal/cli/commands/preset.py b/src/dstack/_internal/cli/commands/preset.py deleted file mode 100644 index 0d00678697..0000000000 --- a/src/dstack/_internal/cli/commands/preset.py +++ /dev/null @@ -1,66 +0,0 @@ -import argparse - -from dstack._internal.cli.commands import APIBaseCommand -from dstack._internal.cli.services.completion import EndpointPresetNameCompleter -from dstack._internal.cli.utils.common import confirm_ask, console -from dstack._internal.cli.utils.preset import print_endpoint_presets_table -from dstack._internal.core.errors import ResourceNotExistsError - - -class PresetCommand(APIBaseCommand): - NAME = "preset" - DESCRIPTION = "Manage endpoint presets" - - def _register(self): - super()._register() - self._parser.set_defaults(subfunc=self._list) - subparsers = self._parser.add_subparsers(dest="action") - - list_parser = subparsers.add_parser( - "list", help="List endpoint presets", formatter_class=self._parser.formatter_class - ) - list_parser.set_defaults(subfunc=self._list) - - delete_parser = subparsers.add_parser( - "delete", - help="Delete endpoint presets", - formatter_class=self._parser.formatter_class, - ) - delete_parser.add_argument( - "name", - help="The name of the endpoint preset", - ).completer = EndpointPresetNameCompleter() # type: ignore[attr-defined] - delete_parser.add_argument( - "-y", "--yes", help="Don't ask for confirmation", action="store_true" - ) - delete_parser.set_defaults(subfunc=self._delete) - - def _command(self, args: argparse.Namespace): - super()._command(args) - args.subfunc(args) - - def _list(self, args: argparse.Namespace): - presets = self.api.client.endpoint_presets.list(self.api.project) - print_endpoint_presets_table(presets) - - def _delete(self, args: argparse.Namespace): - presets = self.api.client.endpoint_presets.list(self.api.project) - if args.name not in {preset.name for preset in presets}: - console.print(f"Endpoint preset [code]{args.name}[/] does not exist") - exit(1) - - if not args.yes and not confirm_ask(f"Delete the endpoint preset [code]{args.name}[/]?"): - console.print("\nExiting...") - return - - try: - with console.status("Deleting endpoint preset..."): - self.api.client.endpoint_presets.delete( - project_name=self.api.project, - names=[args.name], - ) - except ResourceNotExistsError: - console.print(f"Endpoint preset [code]{args.name}[/] does not exist") - exit(1) - - console.print(f"Endpoint preset [code]{args.name}[/] deleted") diff --git a/src/dstack/_internal/cli/main.py b/src/dstack/_internal/cli/main.py index ed58dc1360..335f7693f1 100644 --- a/src/dstack/_internal/cli/main.py +++ b/src/dstack/_internal/cli/main.py @@ -19,7 +19,6 @@ from dstack._internal.cli.commands.logs import LogsCommand from dstack._internal.cli.commands.metrics import MetricsCommand from dstack._internal.cli.commands.offer import OfferCommand -from dstack._internal.cli.commands.preset import PresetCommand from dstack._internal.cli.commands.project import ProjectCommand from dstack._internal.cli.commands.ps import PsCommand from dstack._internal.cli.commands.run import RunCommand @@ -77,7 +76,6 @@ def main(): GatewayCommand.register(subparsers) InitCommand.register(subparsers) OfferCommand.register(subparsers) - PresetCommand.register(subparsers) LoginCommand.register(subparsers) LogsCommand.register(subparsers) MetricsCommand.register(subparsers) diff --git a/src/dstack/_internal/cli/services/completion.py b/src/dstack/_internal/cli/services/completion.py index 754069cc2c..ca5353c9b7 100644 --- a/src/dstack/_internal/cli/services/completion.py +++ b/src/dstack/_internal/cli/services/completion.py @@ -60,13 +60,6 @@ def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.runs.list(self.all)] -class RunOrEndpointNameCompleter(BaseAPINameCompleter): - def fetch_resource_names(self, api: Client) -> Iterable[str]: - run_names = [r.name for r in api.runs.list(all=True)] - endpoint_names = [r.name for r in api.client.endpoints.list(api.project)] - return sorted(set(run_names + endpoint_names)) - - class FleetNameCompleter(BaseAPINameCompleter): def fetch_resource_names(self, api: Client) -> Iterable[str]: return [r.name for r in api.client.fleets.list(api.project)] diff --git a/src/dstack/_internal/cli/services/configurators/endpoint.py b/src/dstack/_internal/cli/services/configurators/endpoint.py index 3a7da0193b..d61f16268d 100644 --- a/src/dstack/_internal/cli/services/configurators/endpoint.py +++ b/src/dstack/_internal/cli/services/configurators/endpoint.py @@ -19,7 +19,7 @@ ) from dstack._internal.cli.utils.endpoint import get_endpoints_table from dstack._internal.cli.utils.rich import MultiItemStatus -from dstack._internal.core.errors import ResourceNotExistsError +from dstack._internal.core.errors import ConfigurationError, ResourceNotExistsError from dstack._internal.core.models.configurations import ApplyConfigurationType from dstack._internal.core.models.endpoints import ( AnyEndpointProvisioningPlan, @@ -83,7 +83,7 @@ def apply_configuration( console.print("Use --force to apply anyway.") return else: - # TODO: Replace v1 stop/delete/recreate with endpoint in-place update + # TODO: Replace v1 stop/recreate with endpoint in-place update # via service rolling deployment once endpoint versioning exists. console.print( f"Endpoint [code]{current_resource.name}[/] already exists." @@ -110,19 +110,21 @@ def apply_configuration( return if current_resource is not None: - with console.status("Deleting existing endpoint..."): - self.api.client.endpoints.delete( + with console.status("Stopping existing endpoint..."): + self.api.client.endpoints.stop( project_name=self.api.project, names=[current_resource.name], ) while True: try: - self.api.client.endpoints.get( + endpoint = self.api.client.endpoints.get( project_name=self.api.project, name=current_resource.name, ) except ResourceNotExistsError: break + if endpoint.status.is_finished(): + break time.sleep(1) if stop_run_name is not None: @@ -150,27 +152,10 @@ def delete_configuration( configuration_path: str, command_args: argparse.Namespace, ): - if conf.name is None: - console.print("[error]Configuration specifies no endpoint to delete[/]") - exit(1) - - try: - self.api.client.endpoints.get( - project_name=self.api.project, - name=conf.name, - ) - except ResourceNotExistsError: - console.print(f"Endpoint [code]{conf.name}[/] does not exist") - exit(1) - - if not command_args.yes and not confirm_ask(f"Delete the endpoint [code]{conf.name}[/]?"): - console.print("\nExiting...") - return - - with console.status("Deleting endpoint..."): - self.api.client.endpoints.delete(project_name=self.api.project, names=[conf.name]) - - console.print(f"Endpoint [code]{conf.name}[/] deleted") + raise ConfigurationError( + "`dstack delete` does not support endpoint configurations. " + "Use `dstack endpoint stop `." + ) @classmethod def register_args(cls, parser: argparse.ArgumentParser): @@ -234,7 +219,11 @@ def _apply_profile(conf: EndpointConfiguration, profile: Profile): def _is_endpoint_apply_finished(endpoint: Endpoint) -> bool: - return endpoint.status in (EndpointStatus.RUNNING, EndpointStatus.FAILED) + return endpoint.status in ( + EndpointStatus.RUNNING, + EndpointStatus.STOPPED, + EndpointStatus.FAILED, + ) def _get_apply_status(endpoint: Endpoint) -> str: @@ -265,8 +254,6 @@ def th(s: str) -> str: if plan.configuration_path is not None: props.add_row(th("Configuration"), plan.configuration_path) props.add_row(th("Type"), plan.configuration.type) - props.add_row(th("Endpoint"), plan.configuration.name or "(generated)") - props.add_row(th("Resources"), _format_resources(plan)) props.add_row(th("Spot policy"), _format_spot_policy(plan)) props.add_row(th("Max price"), _format_max_price(plan)) props.add_row(th("Preset policy"), plan.preset_policy.value) @@ -287,6 +274,8 @@ def th(s: str) -> str: console.print(f"[{style}]{plan.provisioning_plan.reason}[/]") console.print() elif isinstance(plan.provisioning_plan, EndpointProvisioningPlanAgent): + # TODO: Consider showing initial candidate offers for agent provisioning as + # non-binding context. Do not present them as selected resources/final hardware. if plan.provisioning_plan.reason is not None: console.print(f"[warning]{plan.provisioning_plan.reason}[/]") console.print() @@ -318,21 +307,6 @@ def _get_non_terminal_current_resource(plan: EndpointPlan) -> Endpoint | None: return plan.current_resource -def _format_resources(plan: EndpointPlan) -> str: - if not isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): - return "-" - resources = [ - (job_offers.replica_group, job_offers.resources.pretty_format()) - for job_offers in plan.provisioning_plan.job_offers - ] - if not resources: - return "-" - resource_values = {resource for _, resource in resources} - if len(resource_values) == 1: - return resources[0][1] - return "\n".join(f"{replica_group}: {resource}" for replica_group, resource in resources) - - def _format_spot_policy(plan: EndpointPlan) -> str: if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): values = {job_offers.spot for job_offers in plan.provisioning_plan.job_offers} diff --git a/src/dstack/_internal/cli/utils/endpoint.py b/src/dstack/_internal/cli/utils/endpoint.py index c1507fce19..a61e737230 100644 --- a/src/dstack/_internal/cli/utils/endpoint.py +++ b/src/dstack/_internal/cli/utils/endpoint.py @@ -26,7 +26,7 @@ def filter_endpoints_for_listing( latest_finished = None filtered = [] for endpoint in endpoints: - if endpoint.status.is_finished(): + if _is_endpoint_finished(endpoint): if not include_latest_finished: continue if latest_finished is None: @@ -63,7 +63,13 @@ def th(value: str) -> str: table.add_row(th("User"), endpoint.user) table.add_row(th("Endpoint"), endpoint.name) table.add_row(th("Model"), endpoint.configuration.model) - table.add_row(th("Status"), _format_endpoint_status(endpoint.status, endpoint.status_message)) + table.add_row( + th("Status"), + _format_endpoint_status( + endpoint.status, + endpoint.status_message, + ), + ) table.add_row(th("Run"), endpoint.run_name or "-") table.add_row(th("URL"), endpoint.url or "-") table.add_row(th("Created"), format_date(endpoint.created_at)) @@ -92,7 +98,10 @@ def get_endpoints_table( row = { "NAME": endpoint.name, "MODEL": endpoint.configuration.model, - "STATUS": _format_endpoint_status(endpoint.status, endpoint.status_message), + "STATUS": _format_endpoint_status( + endpoint.status, + endpoint.status_message, + ), "RUN": endpoint.run_name or "-", "URL": endpoint.url or "-", "CREATED": format_date(endpoint.created_at), @@ -110,9 +119,10 @@ def _format_endpoint_status( color_map = { EndpointStatus.SUBMITTED: "grey", EndpointStatus.PROVISIONING: "deep_sky_blue1", - EndpointStatus.AGENTING: "medium_purple1", + EndpointStatus.CLAUDING: "medium_purple1", EndpointStatus.RUNNING: "sea_green3", - EndpointStatus.ACTIVE: "sea_green3", + EndpointStatus.STOPPING: "deep_sky_blue1", + EndpointStatus.STOPPED: "grey62", EndpointStatus.FAILED: "indian_red1", } if status_value == "no offers": @@ -128,8 +138,6 @@ def _get_endpoint_status_value( status: EndpointStatus, status_message: Optional[str], ) -> str: - if status == EndpointStatus.ACTIVE: - return EndpointStatus.RUNNING.value if status == EndpointStatus.FAILED: failure_reason = _get_endpoint_failure_reason(status_message) if failure_reason is not None: @@ -137,6 +145,10 @@ def _get_endpoint_status_value( return status.value +def _is_endpoint_finished(endpoint: Endpoint) -> bool: + return endpoint.status.is_finished() + + def _get_endpoint_failure_reason(status_message: Optional[str]) -> Optional[str]: if status_message is None: return None diff --git a/src/dstack/_internal/cli/utils/preset.py b/src/dstack/_internal/cli/utils/preset.py index 74d98ef123..e09c095b42 100644 --- a/src/dstack/_internal/cli/utils/preset.py +++ b/src/dstack/_internal/cli/utils/preset.py @@ -19,59 +19,36 @@ def get_endpoint_presets_table(presets: List[EndpointPreset]) -> Table: table.add_column("RESOURCES") for preset in presets: - total_replicas = sum(len(group.tested_resources) for group in preset.replica_spec_groups) - if total_replicas == 1: + if len(preset.replica_spec_groups) == 1: add_row_from_dict( table, { "NAME": preset.name, "MODEL": preset.model, - "RESOURCES": _format_replica_resources( - preset.replica_spec_groups[0].tested_resources[0] - ), + "RESOURCES": _format_resources(preset.replica_spec_groups[0].resources), }, ) continue add_row_from_dict(table, {"NAME": preset.name, "MODEL": preset.model}) - show_group = len(preset.replica_spec_groups) > 1 - last_group_index = None - for group_index, group in enumerate(preset.replica_spec_groups): - for replica_num, replica_spec in enumerate(group.tested_resources): - add_row_from_dict( - table, - { - "NAME": _format_replica_name( - group_index=group_index, - group_name=group.name, - replica_num=replica_num, - show_group=show_group, - last_group_index=last_group_index, - ), - "RESOURCES": _format_replica_resources(replica_spec), - }, - ) - last_group_index = group_index + for group in preset.replica_spec_groups: + add_row_from_dict( + table, + { + "NAME": f" group={group.name}", + "RESOURCES": _format_resources(group.resources), + }, + ) return table -def _format_replica_name( - *, - group_index: int, - group_name: str, - replica_num: int, - show_group: bool, - last_group_index: int | None, -) -> str: - if not show_group: - return f" replica={replica_num}" - if group_index != last_group_index: - return f" group={group_name} replica={replica_num}" - padding_width = 3 + len(f"group={group_name}") + 1 - return f"{' ' * padding_width}replica={replica_num}" - - -def _format_replica_resources(resources) -> str: +def _format_resources(resources) -> str: formatted = resources.pretty_format() - if resources.gpu is not None and resources.gpu.count.min == 0 and resources.gpu.count.max == 0: - return formatted.replace(" gpu=0", "").replace("gpu=0", "-") + if resources.gpu is not None and resources.gpu.count.min == 0: + if resources.gpu.count.max in (None, 0): + return ( + formatted.replace(" gpu=0..", "") + .replace(" gpu=0", "") + .replace("gpu=0..", "-") + .replace("gpu=0", "-") + ) return formatted diff --git a/src/dstack/_internal/core/models/endpoint_presets.py b/src/dstack/_internal/core/models/endpoint_presets.py index a6340ec738..78a4403397 100644 --- a/src/dstack/_internal/core/models/endpoint_presets.py +++ b/src/dstack/_internal/core/models/endpoint_presets.py @@ -1,4 +1,5 @@ from dstack._internal.core.models.common import CoreModel +from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.resources import ResourcesSpec @@ -16,3 +17,7 @@ class EndpointPreset(CoreModel): name: str model: str replica_spec_groups: list[EndpointPresetReplicaSpecGroup] + + +class EndpointPresetDetails(EndpointPreset): + service: ServiceConfiguration diff --git a/src/dstack/_internal/core/models/endpoints.py b/src/dstack/_internal/core/models/endpoints.py index 9d3289c047..15ac989b0a 100644 --- a/src/dstack/_internal/core/models/endpoints.py +++ b/src/dstack/_internal/core/models/endpoints.py @@ -19,15 +19,15 @@ class EndpointStatus(str, Enum): SUBMITTED = "submitted" PROVISIONING = "provisioning" - AGENTING = "agenting" + CLAUDING = "clauding" RUNNING = "running" - # Legacy status used by early endpoint prototypes. New code writes RUNNING. - ACTIVE = "active" + STOPPING = "stopping" + STOPPED = "stopped" FAILED = "failed" @classmethod def finished_statuses(cls) -> list["EndpointStatus"]: - return [cls.FAILED] + return [cls.STOPPED, cls.FAILED] def is_finished(self) -> bool: return self in self.finished_statuses() @@ -94,8 +94,6 @@ class Endpoint(CoreModel): last_processed_at: datetime status: EndpointStatus status_message: Optional[str] = None - deleted: bool - deleted_at: Optional[datetime] = None run_name: Optional[str] = None url: Optional[str] = None error: Optional[str] = None diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py index 701a2a170f..923a824fcf 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -19,13 +19,11 @@ ServiceSpec, ) from dstack._internal.server.background.pipeline_tasks.base import ( - NOW_PLACEHOLDER, Fetcher, Heartbeater, ItemUpdateMap, Pipeline, PipelineItem, - UpdateMapDateTime, Worker, log_lock_token_changed_after_processing, log_lock_token_mismatch, @@ -35,7 +33,6 @@ ) from dstack._internal.server.db import get_db, get_session_ctx from dstack._internal.server.models import EndpointModel, ProjectModel, RunModel, UserModel -from dstack._internal.server.services import events from dstack._internal.server.services import runs as runs_services from dstack._internal.server.services.endpoints import ( emit_endpoint_status_change_event, @@ -67,7 +64,6 @@ @dataclass class EndpointPipelineItem(PipelineItem): status: EndpointStatus - to_be_deleted: bool class EndpointPipeline(Pipeline[EndpointPipelineItem]): @@ -158,19 +154,15 @@ async def fetch(self, limit: int) -> list[EndpointPipelineItem]: res = await session.execute( select(EndpointModel) .where( - or_( - EndpointModel.status.in_( - [ - EndpointStatus.SUBMITTED, - EndpointStatus.PROVISIONING, - EndpointStatus.AGENTING, - EndpointStatus.RUNNING, - EndpointStatus.ACTIVE, - ] - ), - EndpointModel.to_be_deleted == True, + EndpointModel.status.in_( + [ + EndpointStatus.SUBMITTED, + EndpointStatus.PROVISIONING, + EndpointStatus.CLAUDING, + EndpointStatus.RUNNING, + EndpointStatus.STOPPING, + ] ), - EndpointModel.deleted == False, or_( EndpointModel.last_processed_at <= now - self._min_processing_interval, EndpointModel.last_processed_at == EndpointModel.created_at, @@ -193,7 +185,6 @@ async def fetch(self, limit: int) -> list[EndpointPipelineItem]: EndpointModel.lock_token, EndpointModel.lock_expires_at, EndpointModel.status, - EndpointModel.to_be_deleted, ) ) ) @@ -214,7 +205,6 @@ async def fetch(self, limit: int) -> list[EndpointPipelineItem]: lock_token=lock_token, prev_lock_expired=prev_lock_expired, status=endpoint_model.status, - to_be_deleted=endpoint_model.to_be_deleted, ) ) await session.commit() @@ -241,12 +231,7 @@ async def process(self, item: EndpointPipelineItem): log_lock_token_mismatch(logger, item) return - if endpoint_model.to_be_deleted: - result = await _process_to_be_deleted_endpoint( - endpoint_model=endpoint_model, - pipeline_hinter=self._pipeline_hinter, - ) - elif endpoint_model.status == EndpointStatus.SUBMITTED: + if endpoint_model.status == EndpointStatus.SUBMITTED: result = await _process_submitted_endpoint( endpoint_model=endpoint_model, pipeline_hinter=self._pipeline_hinter, @@ -256,12 +241,17 @@ async def process(self, item: EndpointPipelineItem): endpoint_model=endpoint_model, pipeline_hinter=self._pipeline_hinter, ) - elif endpoint_model.status == EndpointStatus.AGENTING: - result = await _process_agenting_endpoint( + elif endpoint_model.status == EndpointStatus.CLAUDING: + result = await _process_clauding_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=self._pipeline_hinter, + ) + elif endpoint_model.status == EndpointStatus.STOPPING: + result = await _process_stopping_endpoint( endpoint_model=endpoint_model, pipeline_hinter=self._pipeline_hinter, ) - elif endpoint_model.status in (EndpointStatus.RUNNING, EndpointStatus.ACTIVE): + elif endpoint_model.status == EndpointStatus.RUNNING: result = await _process_running_endpoint(endpoint_model) else: result = _ProcessResult() @@ -322,21 +312,13 @@ async def _apply_process_result( if len(updated_ids) == 0: log_lock_token_changed_after_processing(logger, item) return - if result.update_map.get("deleted"): - events.emit( - session, - "Endpoint deleted", - actor=events.SystemActor(), - targets=[events.Target.from_model(endpoint_model)], - ) - else: - emit_endpoint_status_change_event( - session=session, - endpoint_model=endpoint_model, - old_status=endpoint_model.status, - new_status=update_map.get("status", endpoint_model.status), - status_message=update_map.get("status_message", endpoint_model.status_message), - ) + emit_endpoint_status_change_event( + session=session, + endpoint_model=endpoint_model, + old_status=endpoint_model.status, + new_status=update_map.get("status", endpoint_model.status), + status_message=update_map.get("status_message", endpoint_model.status_message), + ) class _EndpointUpdateMap(ItemUpdateMap, total=False): @@ -344,8 +326,6 @@ class _EndpointUpdateMap(ItemUpdateMap, total=False): status_message: Optional[str] service_run_id: uuid.UUID provisioning_method: Optional[str] - deleted: bool - deleted_at: UpdateMapDateTime @dataclass @@ -383,10 +363,6 @@ async def _process_submitted_endpoint( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: - conflict_result = await _get_active_serving_run_name_conflict(endpoint_model) - if conflict_result is not None: - return conflict_result - try: submission_result = await _submit_endpoint_from_preset( endpoint_id=endpoint_model.id, @@ -416,7 +392,7 @@ async def _process_submitted_endpoint( if _should_provision_with_agent(endpoint_model): logger.info("Provisioning endpoint %s with server agent", endpoint_model.name) update_map = _EndpointUpdateMap( - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, status_message=None, provisioning_method="agent", ) @@ -473,7 +449,7 @@ async def _process_provisioning_endpoint( ) -> _ProcessResult: if endpoint_model.service_run is None: if endpoint_model.provisioning_method == "agent": - return await _process_agenting_endpoint( + return await _process_clauding_endpoint( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) @@ -506,7 +482,7 @@ async def _process_provisioning_endpoint( ) -async def _process_agenting_endpoint( +async def _process_clauding_endpoint( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: @@ -531,7 +507,7 @@ async def _process_agent_verified_endpoint(endpoint_model: EndpointModel) -> _Pr } ) if readiness.model_base_url is None or readiness.model_name is None: - return _ProcessResult(update_map={"status": EndpointStatus.AGENTING}) + return _ProcessResult(update_map={"status": EndpointStatus.CLAUDING}) await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, @@ -562,6 +538,13 @@ async def _provision_endpoint_with_agent( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) + if result.in_progress: + return _ProcessResult( + update_map={ + "status": EndpointStatus.CLAUDING, + "status_message": None, + } + ) if result.error is not None: return _ProcessResult( update_map={ @@ -681,7 +664,7 @@ async def _provision_endpoint_with_agent( if readiness.model_base_url is None or readiness.model_name is None: return _ProcessResult( update_map={ - "status": EndpointStatus.AGENTING, + "status": EndpointStatus.CLAUDING, "status_message": None, "service_run_id": run_model.id, } @@ -736,6 +719,29 @@ async def _process_running_endpoint(endpoint_model: EndpointModel) -> _ProcessRe return _ProcessResult(update_map={"status_message": None}) +async def _process_stopping_endpoint( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: + run_model = endpoint_model.service_run + # TODO: When the Claude agent service exposes cancellation, interrupt a live + # agent process here instead of waiting for it to notice/finish. + if run_model is None or run_model.deleted or run_model.status.is_finished(): + return _get_stopped_result() + if run_model.status != RunStatus.TERMINATING: + logger.info( + "Stopping backing run %s before stopping endpoint %s", + run_model.run_name, + endpoint_model.name, + ) + await _stop_backing_run( + endpoint_model=endpoint_model, + run_name=run_model.run_name, + pipeline_hinter=pipeline_hinter, + ) + return _ProcessResult() + + async def _try_save_agent_endpoint_preset( endpoint_model: EndpointModel, model_name: str, @@ -759,6 +765,7 @@ async def _save_agent_endpoint_preset( try: preset = build_endpoint_preset_from_run(name=preset_name, run_model=run_model) saved_preset = await get_endpoint_preset_service().save_preset( + endpoint_model.project.name, preset, comments=[ "Generated by dstack endpoint agent.", @@ -835,8 +842,6 @@ async def _submit_endpoint_from_preset( select(EndpointModel) .where( EndpointModel.id == endpoint_id, - EndpointModel.deleted == False, - EndpointModel.to_be_deleted == False, EndpointModel.status == EndpointStatus.SUBMITTED, ) .options(joinedload(EndpointModel.project).joinedload(ProjectModel.backends)) @@ -861,6 +866,14 @@ async def _submit_endpoint_from_preset( if preset_planning_result.unprovisionable is not None: unprovisionable_preset_name = preset_planning_result.unprovisionable.preset.name return _PresetSubmissionResult(unprovisionable_preset_name=unprovisionable_preset_name) + conflict_message = await _get_active_run_name_conflict_message( + session=session, + project=endpoint_model.project, + run_name=preset_plan.run_plan.run_spec.run_name, + linked_run_id=endpoint_model.service_run_id, + ) + if conflict_message is not None: + raise ServerClientError(conflict_message) run = await runs_services.apply_plan( session=session, user=endpoint_model.user, @@ -883,6 +896,29 @@ async def _submit_endpoint_from_preset( ) +async def _get_active_run_name_conflict_message( + *, + session, + project: ProjectModel, + run_name: Optional[str], + linked_run_id: Optional[uuid.UUID], +) -> Optional[str]: + if run_name is None: + return None + run_model = await runs_services.get_run_model_by_name( + session=session, + project=project, + run_name=run_name, + ) + if run_model is None: + return None + if linked_run_id == run_model.id: + return None + if run_model.status.is_finished(): + return None + return f"Run name '{run_name}' is taken by an existing run" + + def _get_no_provisioning_path_message( endpoint_model: EndpointModel, unprovisionable_preset_name: Optional[str] = None, @@ -911,43 +947,6 @@ def _get_no_provisioning_path_message( return f"Preset policy create requires the server agent, but {agent_unavailable_reason}" -async def _process_to_be_deleted_endpoint( - endpoint_model: EndpointModel, - pipeline_hinter: PipelineHinterProtocol, -) -> _ProcessResult: - run_model = await _get_backing_run_for_deletion(endpoint_model) - if run_model is None: - return _get_deleted_result() - if not run_model.status.is_finished(): - if run_model.status != RunStatus.TERMINATING: - logger.info( - "Stopping backing run %s before deleting endpoint %s", - run_model.run_name, - endpoint_model.name, - ) - await _stop_backing_run( - endpoint_model=endpoint_model, - run_name=run_model.run_name, - pipeline_hinter=pipeline_hinter, - ) - return _ProcessResult() - - logger.info( - "Deleting finished backing run %s before deleting endpoint %s", - run_model.run_name, - endpoint_model.name, - ) - await _delete_backing_run(endpoint_model=endpoint_model, run_name=run_model.run_name) - return _get_deleted_result() - - -async def _get_backing_run_for_deletion(endpoint_model: EndpointModel) -> Optional[RunModel]: - run_model = endpoint_model.service_run - if run_model is not None and not run_model.deleted: - return run_model - return None - - async def _stop_backing_run( endpoint_model: EndpointModel, run_name: str, @@ -964,20 +963,10 @@ async def _stop_backing_run( ) -async def _delete_backing_run(endpoint_model: EndpointModel, run_name: str) -> None: - async with get_session_ctx() as session: - await runs_services.delete_runs( - session=session, - user=endpoint_model.user, - project=endpoint_model.project, - runs_names=[run_name], - ) - - -def _get_deleted_result() -> _ProcessResult: +def _get_stopped_result() -> _ProcessResult: return _ProcessResult( update_map={ - "deleted": True, - "deleted_at": NOW_PLACEHOLDER, + "status": EndpointStatus.STOPPED, + "status_message": None, } ) diff --git a/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py b/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py index f6f8798aff..ac1444c6f1 100644 --- a/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py +++ b/src/dstack/_internal/server/migrations/versions/2026/07_03_1243_03efb71a1563_add_endpoints.py @@ -40,12 +40,6 @@ def upgrade() -> None: sa.Column( "last_processed_at", dstack._internal.server.models.NaiveDateTime(), nullable=False ), - sa.Column("to_be_deleted", sa.Boolean(), server_default=sa.false(), nullable=False), - sa.Column( - "deletion_requested_at", dstack._internal.server.models.NaiveDateTime(), nullable=True - ), - sa.Column("deleted", sa.Boolean(), nullable=False), - sa.Column("deleted_at", dstack._internal.server.models.NaiveDateTime(), nullable=True), sa.Column( "lock_expires_at", dstack._internal.server.models.NaiveDateTime(), nullable=True ), @@ -64,14 +58,13 @@ def upgrade() -> None: ["user_id"], ["users.id"], name=op.f("fk_endpoints_user_id_users"), ondelete="CASCADE" ), sa.PrimaryKeyConstraint("id", name=op.f("pk_endpoints")), + sa.UniqueConstraint("project_id", "name", name="uq_endpoints_project_id_name"), ) with op.batch_alter_table("endpoints", schema=None) as batch_op: batch_op.create_index( "ix_endpoints_pipeline_fetch_q", [sa.literal_column("last_processed_at ASC")], unique=False, - postgresql_where=sa.text("deleted IS FALSE"), - sqlite_where=sa.text("deleted = 0"), ) batch_op.create_index(batch_op.f("ix_endpoints_status"), ["status"], unique=False) @@ -119,8 +112,6 @@ def downgrade() -> None: batch_op.drop_index(batch_op.f("ix_endpoints_status")) batch_op.drop_index( "ix_endpoints_pipeline_fetch_q", - postgresql_where=sa.text("deleted IS FALSE"), - sqlite_where=sa.text("deleted = 0"), ) op.drop_table("endpoints") diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 979578b424..29595a0c46 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -1031,17 +1031,12 @@ class EndpointModel(PipelineModelMixin, BaseModel): last_processed_at: Mapped[datetime] = mapped_column( NaiveDateTime, default=get_current_datetime ) - to_be_deleted: Mapped[bool] = mapped_column(Boolean, server_default=false()) - deletion_requested_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) - deleted: Mapped[bool] = mapped_column(Boolean, default=False) - deleted_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) __table_args__ = ( + UniqueConstraint("project_id", "name", name="uq_endpoints_project_id_name"), Index( "ix_endpoints_pipeline_fetch_q", last_processed_at.asc(), - postgresql_where=deleted == false(), - sqlite_where=deleted == false(), ), ) diff --git a/src/dstack/_internal/server/routers/endpoints.py b/src/dstack/_internal/server/routers/endpoints.py index b7f891b694..883109d572 100644 --- a/src/dstack/_internal/server/routers/endpoints.py +++ b/src/dstack/_internal/server/routers/endpoints.py @@ -5,20 +5,24 @@ import dstack._internal.server.services.endpoints as endpoints_services from dstack._internal.core.errors import ResourceNotExistsError -from dstack._internal.core.models.endpoint_presets import EndpointPreset +from dstack._internal.core.models.endpoint_presets import EndpointPreset, EndpointPresetDetails from dstack._internal.core.models.endpoints import Endpoint, EndpointPlan from dstack._internal.server.db import get_session from dstack._internal.server.models import ProjectModel, UserModel -from dstack._internal.server.schemas.endpoint_presets import DeleteEndpointPresetsRequest +from dstack._internal.server.schemas.endpoint_presets import ( + DeleteEndpointPresetsRequest, + GetEndpointPresetRequest, +) from dstack._internal.server.schemas.endpoints import ( CreateEndpointRequest, - DeleteEndpointsRequest, GetEndpointPlanRequest, GetEndpointRequest, ListEndpointsRequest, + StopEndpointsRequest, ) from dstack._internal.server.security.permissions import Authenticated, ProjectMember from dstack._internal.server.services.endpoints.presets import ( + endpoint_preset_to_api_details, endpoint_preset_to_api_model, get_endpoint_preset_service, ) @@ -63,7 +67,10 @@ async def list_project_endpoints( ): _, project = user_project return CustomORJSONResponse( - await endpoints_services.list_project_endpoints(session=session, project=project) + await endpoints_services.list_project_endpoints( + session=session, + project=project, + ) ) @@ -121,15 +128,15 @@ async def create_endpoint( ) -@project_router.post("/delete", summary="Delete endpoints") -async def delete_endpoints( - body: DeleteEndpointsRequest, +@project_router.post("/stop", summary="Stop endpoints") +async def stop_endpoints( + body: StopEndpointsRequest, session: AsyncSession = Depends(get_session), user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), pipeline_hinter: PipelineHinterProtocol = Depends(get_pipeline_hinter), ): user, project = user_project - await endpoints_services.delete_endpoints( + await endpoints_services.stop_endpoints( session=session, project=project, names=body.names, @@ -144,20 +151,34 @@ async def delete_endpoints( async def list_endpoint_presets( user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), ): - _, _ = user_project - presets = await get_endpoint_preset_service().list_presets() + _, project = user_project + presets = await get_endpoint_preset_service().list_presets(project.name) return CustomORJSONResponse([endpoint_preset_to_api_model(preset) for preset in presets]) +@project_router.post( + "/presets/get", summary="Get endpoint preset", response_model=EndpointPresetDetails +) +async def get_endpoint_preset( + body: GetEndpointPresetRequest, + user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), +): + _, project = user_project + preset = await get_endpoint_preset_service().get_preset(project.name, body.name) + if preset is None: + raise ResourceNotExistsError() + return CustomORJSONResponse(endpoint_preset_to_api_details(preset)) + + @project_router.post("/presets/delete", summary="Delete endpoint presets") async def delete_endpoint_presets( body: DeleteEndpointPresetsRequest, user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), ): - _, _ = user_project + _, project = user_project preset_service = get_endpoint_preset_service() try: for name in body.names: - await preset_service.delete_preset(name) + await preset_service.delete_preset(project.name, name) except FileNotFoundError: raise ResourceNotExistsError() diff --git a/src/dstack/_internal/server/schemas/endpoint_presets.py b/src/dstack/_internal/server/schemas/endpoint_presets.py index 92ec58887f..9e46813baa 100644 --- a/src/dstack/_internal/server/schemas/endpoint_presets.py +++ b/src/dstack/_internal/server/schemas/endpoint_presets.py @@ -3,5 +3,9 @@ from dstack._internal.core.models.common import CoreModel +class GetEndpointPresetRequest(CoreModel): + name: str + + class DeleteEndpointPresetsRequest(CoreModel): names: List[str] diff --git a/src/dstack/_internal/server/schemas/endpoints.py b/src/dstack/_internal/server/schemas/endpoints.py index d3ecbf780c..0cc847d4ae 100644 --- a/src/dstack/_internal/server/schemas/endpoints.py +++ b/src/dstack/_internal/server/schemas/endpoints.py @@ -30,5 +30,5 @@ class CreateEndpointRequest(CoreModel): configuration: EndpointConfiguration -class DeleteEndpointsRequest(CoreModel): +class StopEndpointsRequest(CoreModel): names: List[str] diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py index 6d9dac2867..7a571a9fb9 100644 --- a/src/dstack/_internal/server/services/endpoints/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -141,7 +141,7 @@ async def list_projects_endpoint_models( return [] filters: list[Any] = [EndpointModel.project_id.in_(p.id for p in projects)] if only_active: - filters.append(EndpointModel.deleted == False) + filters.append(EndpointModel.status.not_in(EndpointStatus.finished_statuses())) if prev_created_at is not None: if ascending: if prev_id is None: @@ -190,7 +190,9 @@ async def list_project_endpoints( names: Optional[List[str]] = None, ) -> List[Endpoint]: endpoint_models = await list_project_endpoint_models( - session=session, project=project, names=names + session=session, + project=project, + names=names, ) return [endpoint_model_to_endpoint(e) for e in endpoint_models] @@ -199,13 +201,10 @@ async def list_project_endpoint_models( session: AsyncSession, project: ProjectModel, names: Optional[List[str]] = None, - include_deleted: bool = False, ) -> List[EndpointModel]: filters = [EndpointModel.project_id == project.id] if names is not None: filters.append(EndpointModel.name.in_(names)) - if not include_deleted: - filters.append(EndpointModel.deleted == False) res = await session.execute( select(EndpointModel) .where(*filters) @@ -217,10 +216,14 @@ async def list_project_endpoint_models( async def get_endpoint_by_name( - session: AsyncSession, project: ProjectModel, name: str + session: AsyncSession, + project: ProjectModel, + name: str, ) -> Optional[Endpoint]: endpoint_model = await get_project_endpoint_model_by_name( - session=session, project=project, name=name + session=session, + project=project, + name=name, ) if endpoint_model is None: return None @@ -231,14 +234,11 @@ async def get_project_endpoint_model_by_name( session: AsyncSession, project: ProjectModel, name: str, - include_deleted: bool = False, ) -> Optional[EndpointModel]: filters = [ EndpointModel.name == name, EndpointModel.project_id == project.id, ] - if not include_deleted: - filters.append(EndpointModel.deleted == False) res = await session.execute( select(EndpointModel) .where(*filters) @@ -415,8 +415,27 @@ async def create_endpoint( if endpoint_model is not None: if not endpoint_model.status.is_finished(): raise ResourceExistsError() - endpoint_model.deleted = True - endpoint_model.deleted_at = now + endpoint_model.user = user + endpoint_model.service_run_id = None + endpoint_model.service_run = None + endpoint_model.configuration = configuration.json() + endpoint_model.status = EndpointStatus.SUBMITTED + endpoint_model.status_message = None + endpoint_model.provisioning_method = None + endpoint_model.created_at = now + endpoint_model.last_processed_at = now + endpoint_model.lock_expires_at = None + endpoint_model.lock_token = None + endpoint_model.lock_owner = None + events.emit( + session, + message=f"Endpoint submitted. Status: {endpoint_model.status.upper()}", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(endpoint_model)], + ) + await session.commit() + pipeline_hinter.hint_fetch(EndpointModel.__name__) + return endpoint_model_to_endpoint(endpoint_model) else: configuration.name = await generate_endpoint_name(session=session, project=project) @@ -442,7 +461,7 @@ async def create_endpoint( return endpoint_model_to_endpoint(endpoint_model) -async def delete_endpoints( +async def stop_endpoints( session: AsyncSession, project: ProjectModel, names: List[str], @@ -454,15 +473,14 @@ async def delete_endpoints( project=project, names=names, ) - now = common.get_current_datetime() for endpoint_model in endpoint_models: - if endpoint_model.to_be_deleted: + if endpoint_model.status.is_finished(): continue - endpoint_model.to_be_deleted = True - endpoint_model.deletion_requested_at = now + endpoint_model.status = EndpointStatus.STOPPING + endpoint_model.status_message = None events.emit( session, - message="Endpoint marked for deletion", + message="Endpoint marked for stopping", actor=events.UserActor.from_user(user), targets=[events.Target.from_model(endpoint_model)], ) @@ -476,11 +494,12 @@ def endpoint_model_to_endpoint(endpoint_model: EndpointModel) -> Endpoint: run_name = None url = None status = endpoint_model.status - if status == EndpointStatus.ACTIVE: - status = EndpointStatus.RUNNING if endpoint_model.service_run is not None and not endpoint_model.service_run.deleted: run_name = endpoint_model.service_run.run_name - if endpoint_model.service_run.service_spec is not None: + if ( + status == EndpointStatus.RUNNING + and endpoint_model.service_run.service_spec is not None + ): service_spec = ServiceSpec.__response__.parse_raw( endpoint_model.service_run.service_spec ) @@ -496,11 +515,13 @@ def endpoint_model_to_endpoint(endpoint_model: EndpointModel) -> Endpoint: last_processed_at=endpoint_model.last_processed_at, status=status, status_message=endpoint_model.status_message, - deleted=endpoint_model.deleted, - deleted_at=endpoint_model.deleted_at, run_name=run_name, url=url, - error=endpoint_model.status_message if status == EndpointStatus.FAILED else None, + error=( + endpoint_model.status_message + if endpoint_model.status == EndpointStatus.FAILED + else None + ), ) @@ -553,7 +574,6 @@ async def generate_endpoint_name(session: AsyncSession, project: ProjectModel) - res = await session.execute( select(EndpointModel.name).where( EndpointModel.project_id == project.id, - EndpointModel.deleted == False, ) ) names = set(res.scalars().all()) diff --git a/src/dstack/_internal/server/services/endpoints/agent/__init__.py b/src/dstack/_internal/server/services/endpoints/agent/__init__.py index 68510cd8f6..3bef1d5043 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/agent/__init__.py @@ -21,6 +21,7 @@ class AgentProvisioningResult: run_name: Optional[str] = None error: Optional[str] = None final_report: Optional[AgentFinalReport] = None + in_progress: bool = False class AgentService(ABC): diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index 020c836335..903064b4a8 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -1,4 +1,5 @@ import asyncio +import enum import json import os import shutil @@ -35,12 +36,8 @@ logger = get_logger(__name__) _RESOURCE_DIR = Path(__file__).parent / "resources" -_PROMPT_RESOURCE_NAMES = [ - "system_prompt.md", - "dstack_cli_and_service_authoring.md", - "recipes_guide.md", - "deployment_harness.md", -] +_ENDPOINT_PROMPT_RESOURCE_NAME = "system_prompt.md" +_AGENT_SKILL_NAMES = ["dstack", "dstack-prototyping"] _INHERITED_ENV_NAMES = [ "PATH", "SSL_CERT_FILE", @@ -57,6 +54,7 @@ _MAX_AGENT_LOG_MESSAGE_CHARS = 4_000 _AGENT_LOG_BATCH_SIZE = 1 _AGENT_PROGRESS_LOG_NAME = "progress.jsonl" +_AGENT_PROCESS_STATE_NAME = "agent_process.json" _AGENT_PROGRESS_POLL_SECONDS = 1.0 _CLAUDE_AGENT_TOOLS = "Bash,Read,Write,Edit,WebFetch,WebSearch,StructuredOutput" @@ -111,6 +109,24 @@ async def provision_endpoint( error=f"Failed to prepare endpoint agent workspace: {e}" ) + existing_report = _load_final_report(workspace) + if existing_report is not None: + workspace.artifacts.record_report(existing_report) + return AgentProvisioningResult( + run_id=existing_report.run_id, + run_name=existing_report.run_name, + final_report=existing_report, + ) + + running_pid = _get_running_agent_process_pid(workspace) + if running_pid is not None: + logger.info( + "Endpoint agent process %s is already running for endpoint %s", + running_pid, + endpoint_model.name, + ) + return AgentProvisioningResult(in_progress=True) + runner_result = await self._runner(workspace, _build_agent_request(workspace)) if runner_result.error is not None: workspace.artifacts.record_error(runner_result.error) @@ -294,6 +310,7 @@ def _prepare_workspace( endpoint_name=endpoint_model.name, ), ) + _install_agent_skills(workspace) workspace.artifacts.initialize() return workspace @@ -305,6 +322,7 @@ def __init__(self, workspace: _AgentWorkspace) -> None: def initialize(self) -> None: self._workspace.work_dir.mkdir(parents=True, exist_ok=True) + self._command_counter = self._get_last_command_output_num() self._update_agent_state(phase="starting") for filename in [ "sources.jsonl", @@ -452,6 +470,18 @@ def _append_jsonl(self, filename: str, record: dict[str, Any]) -> None: json.dumps(_redact(record, self._workspace.redacted_values), default=str) + "\n" ) + def _get_last_command_output_num(self) -> int: + output_dir = self._workspace.work_dir / "command-output" + if not output_dir.exists(): + return 0 + nums = [] + for path in output_dir.glob("*.txt"): + try: + nums.append(int(path.stem)) + except ValueError: + continue + return max(nums, default=0) + def _write_cli_config( *, @@ -514,33 +544,53 @@ def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: def _build_prompt(workspace: _AgentWorkspace) -> str: - return f"""Deploy endpoint {workspace.endpoint_name!r} for model {workspace.model!r}. + return f"""{_load_endpoint_prompt()} -Use the real `dstack` CLI available in this process. Do not use any custom dstack APIs -or wait for any server-side helper tool. The embedded guidance below is the dstack and -dstack-development playbook for this run. +Deploy endpoint {workspace.endpoint_name!r} for model {workspace.model!r}. {workspace.endpoint_constraints} +Bundled Claude Code skills are installed in `.claude/skills`. Load and follow: +- `/dstack` +- `/dstack-prototyping` + Required final service: - service YAML type: service - service YAML must set `model: {workspace.model}` +- service run name must not equal the endpoint name `{workspace.endpoint_name}` +- for sequential service attempts, prefer `{workspace.endpoint_name}-1`, `{workspace.endpoint_name}-2`, etc. - submit runs detached with `dstack apply -f -y -d` - use concise, unique run names that are useful for debugging - the successful final report must include the verified service run `run_id` -{_load_prompt_resources()} - When you have a terminal result, write `final_report.json` in the workspace with the same fields requested by the JSON schema, then return only the structured final report. """ -def _load_prompt_resources() -> str: - return "\n\n".join( - (_RESOURCE_DIR / resource_name).read_text(encoding="utf-8").strip() - for resource_name in _PROMPT_RESOURCE_NAMES - ) +def _load_endpoint_prompt() -> str: + return (_RESOURCE_DIR / _ENDPOINT_PROMPT_RESOURCE_NAME).read_text(encoding="utf-8").strip() + + +def _install_agent_skills(workspace: _AgentWorkspace) -> None: + source_dir = _get_packaged_skills_dir() + target_dir = workspace.work_dir / ".claude" / "skills" + if target_dir.exists(): + shutil.rmtree(target_dir) + target_dir.mkdir(parents=True, exist_ok=True) + for skill_name in _AGENT_SKILL_NAMES: + source = source_dir / skill_name + if not (source / "SKILL.md").is_file(): + raise FileNotFoundError(f"Missing endpoint agent skill: {skill_name}") + shutil.copytree(source, target_dir / skill_name) + + +def _get_packaged_skills_dir() -> Path: + for parent in Path(__file__).resolve().parents: + candidate = parent / "skills" + if (candidate / "dstack" / "SKILL.md").is_file(): + return candidate + raise FileNotFoundError("Could not find packaged dstack skills") def _format_endpoint_constraints( @@ -601,6 +651,8 @@ def _format_endpoint_constraints( def _format_constraint_value(value: Any) -> str: + if isinstance(value, enum.Enum): + return str(value.value) if hasattr(value, "name") and getattr(value, "project", None) is None: return str(value.name) if hasattr(value, "json"): @@ -615,13 +667,10 @@ async def _run_agent_in_subprocess( request: dict[str, Any], ) -> _AgentRunnerResult: cmd = _build_claude_command(request) - await _write_agent_log( - workspace, - _format_agent_start_log(request), - ) workspace.artifacts.mark_running() progress_tailer = _AgentProgressLogTailer(workspace) progress_task = asyncio.create_task(progress_tailer.run()) + proc: asyncio.subprocess.Process | None = None try: proc = await asyncio.create_subprocess_exec( *cmd, @@ -630,6 +679,7 @@ async def _run_agent_in_subprocess( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) + _write_agent_process_state(workspace, proc.pid) assert proc.stdout is not None assert proc.stderr is not None stdout_task = asyncio.create_task(_read_agent_stdout(proc.stdout, workspace)) @@ -644,6 +694,8 @@ async def _run_agent_in_subprocess( with suppress(asyncio.CancelledError): await progress_task await progress_tailer.flush(include_partial=True) + if proc is not None and proc.returncode is not None: + _clear_agent_process_state(workspace, proc.pid) process_output = _merge_process_outputs(stdout_output, stderr_output) _write_trace_record( workspace, @@ -652,7 +704,6 @@ async def _run_agent_in_subprocess( "returncode": returncode, }, ) - await _write_agent_log(workspace, _format_agent_exit_log(returncode)) await _flush_agent_logs(workspace) if process_output.report_data is None: process_output.report_data = _load_final_report_artifact(workspace) @@ -759,7 +810,9 @@ async def _read_agent_stream( class _AgentProgressLogTailer: def __init__(self, workspace: _AgentWorkspace) -> None: self._workspace = workspace - self._offset = 0 + self._offset = ( + workspace.progress_path.stat().st_size if workspace.progress_path.exists() else 0 + ) self._partial = "" async def run(self) -> None: @@ -870,20 +923,6 @@ async def _flush_agent_logs(workspace: _AgentWorkspace) -> None: await workspace.log_writer.flush() -def _format_agent_start_log(request: dict[str, Any]) -> str: - options = request["options"] - parts = [f"Starting endpoint provisioning agent ({options['model']})"] - if options["max_budget"] is not None: - parts.append(f"max budget ${options['max_budget']}") - return ", ".join(parts) - - -def _format_agent_exit_log(returncode: Optional[int]) -> str: - if returncode == 0: - return "Endpoint provisioning agent finished" - return f"Endpoint provisioning agent exited with code {returncode}" - - def _truncate_log_message(message: str) -> str: if len(message) <= _MAX_AGENT_LOG_MESSAGE_CHARS: return message @@ -917,6 +956,72 @@ def _load_final_report_artifact(workspace: _AgentWorkspace) -> Optional[dict[str return parsed +def _load_final_report(workspace: _AgentWorkspace) -> Optional[AgentFinalReport]: + report_data = _load_final_report_artifact(workspace) + if report_data is None: + return None + try: + return AgentFinalReport.parse_obj(report_data) + except ValidationError: + logger.warning( + "Endpoint agent final_report.json does not match the report schema: %s", + workspace.work_dir / "final_report.json", + exc_info=True, + ) + return None + + +def _write_agent_process_state(workspace: _AgentWorkspace, pid: int) -> None: + path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME + data = { + "pid": pid, + "started_at": _utcnow_iso(), + } + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def _clear_agent_process_state(workspace: _AgentWorkspace, pid: int) -> None: + path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME + if not path.exists(): + return + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + path.unlink(missing_ok=True) + return + if data.get("pid") == pid: + path.unlink(missing_ok=True) + + +def _get_running_agent_process_pid(workspace: _AgentWorkspace) -> Optional[int]: + path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + path.unlink(missing_ok=True) + return None + pid = data.get("pid") + if not isinstance(pid, int): + path.unlink(missing_ok=True) + return None + if _is_process_running(pid): + return pid + path.unlink(missing_ok=True) + return None + + +def _is_process_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + def _write_trace_record(workspace: _AgentWorkspace, data: dict[str, Any]) -> None: if workspace.trace_path is None: return diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md b/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md deleted file mode 100644 index 98f236e359..0000000000 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/deployment_harness.md +++ /dev/null @@ -1,59 +0,0 @@ -# Deployment Harness - -Use this lifecycle: - -1. Understand the model: source URLs, serving framework options, expected memory/disk, and any model-specific launch requirements. -2. Inspect project capacity: use `dstack offer` and/or existing fleets with the endpoint's binding constraints. -3. Pick the fastest credible experiment path: - - If evidence is strong, preview and submit the final service candidate directly. - - If startup commands, images, framework versions, or hardware are uncertain, prototype first. - - Prototypes may be dstack services, tasks, or dev environments when that is the most effective way to test commands interactively. -4. Keep one active candidate at a time where possible. Stop bad candidates before replacing them. -5. Watch status using JSON run state, events, and logs. Use short probes and update workspace artifacts; do not hide progress inside long `sleep` or `until` loops. -6. When the final service reaches `running`, check `dstack run get --json` for job status and probe state. A useful readiness signal is a running job with an HTTP probe `success_streak` greater than zero. Do not wait for a `registered` field if it is absent or null. -7. Get the model URL from `dstack run get --json`, combine it with the configured server URL from `~/.dstack/config.yml` if it is relative, and send a real OpenAI-compatible chat request for the requested model. -8. Once the model request succeeds, write `verification.json`, write `final_report.json`, and return the structured final report immediately. Otherwise return a failed report with the useful evidence and the last candidate state. - -Workspace artifact contract: - -- On startup or resume, inspect `agent_state.json`, `candidates.jsonl`, `verification.json`, and `final_report.json` before submitting anything new. If a previous candidate is already running and verifiable, reuse it instead of creating a duplicate run. -- Keep `agent_state.json` current enough to show the phase. -- Append user-facing major progress events to `progress.jsonl` in real time. Each line must be a small JSON object such as `{"phase":"research","message":"Checking Qwen serving recipes and model requirements"}`. Use this only for milestones: research direction, chosen experiment path, candidate submitted, provisioning observation, verification result, cleanup, or terminal failure. Do not write command output, YAML, secrets, long tables, or raw traces here. -- Add source evidence to `sources.jsonl`. -- Keep `hardware_reasoning.md` reviewable before any paid run. -- Record every spend-capable service, task, or dev-environment candidate in `candidates.jsonl`. -- The server records shell command/tool output in `commands.jsonl` and `command-output/`; use those artifacts when diagnosing. -- On success, write `verification.json` with the final model request evidence before returning the final report. -- Always write `final_report.json` before returning the structured final report. - -Development environments and tasks are allowed for experimentation. Use them when they reduce uncertainty, for example to test an image, install path, model download, launch command, or framework compatibility before committing to the final service. They are not the endpoint: the final verified run reported to the server must be a dstack service. - -For v1, do not attempt P/D disaggregation, router/worker multi-service topologies, autoscaling tuning, load benchmarking, or performance optimization unless they are strictly necessary to make the requested model serve at all. Note when those would be the right next step. - -Hardware behavior: - -- Do not blindly select the cheapest offer. -- Prefer hardware likely to run the serving image reliably: enough VRAM, enough disk, common CUDA-capable NVIDIA GPUs when using CUDA images, and offers without obvious provisioning instability. -- Derive scheduling requirements from the model and serving method before looking at - concrete offers. Preview offers are placement evidence, not the target hardware spec. - Do not copy a preview offer's GPU name, region, instance type, CPU, memory, or disk into - the service YAML unless the model/framework actually requires it or you are intentionally - avoiding a proven failed class of hardware. -- Keep service `resources` as broad as correctness allows: minimum GPU memory/count, - required CPU/memory/disk, tensor-parallel needs, and endpoint/profile constraints. Exact - hardware belongs in the verified run evidence after provisioning, not in the initial plan. -- After a backend no-capacity or supply-constraint failure, do not just retry the same - concrete backend/region/GPU combination. Change the hypothesis or scheduling constraints - so dstack can try materially different viable offers, or stop and report why no credible - alternative remains. -- After the final service is running, re-read `dstack run get --json` and use - the actual latest job submission to identify the backend, region, price, instance type, - and resources that really provisioned. Do not infer final hardware from the service YAML, - the run name, or the first offer shown in a preview. -- If a candidate stays in backend provisioning without logs/events progress after several polls, inspect run JSON/events and any available native backend or SSH/TCP evidence, then stop or fail with evidence rather than looping forever. -- If a backend or dstack provisioning issue is found, create or reference a minimal non-endpoint reproduction in `endpoint-agent-backend-troubleshooting.md` when possible. - -Final report: - -- On success, include the final service run id, final service run name, the exact final service YAML, recipe/source URLs, the actual provisioned hardware from run JSON, and a concise verification summary. -- On failure, include the failure summary and enough evidence for the next iteration to improve the harness. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md b/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md deleted file mode 100644 index c6bf4627d5..0000000000 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/dstack_cli_and_service_authoring.md +++ /dev/null @@ -1,34 +0,0 @@ -# dstack CLI and Service Authoring - -Core CLI rules: - -- Preview before submit with `printf 'n\n' | dstack apply -f ...`. -- Submit runs detached with `dstack apply -f -y -d`. -- Never run `dstack apply` for a run without `-d` in this non-interactive server context. -- Do not use `dstack ps` table text for machine decisions; table output can be truncated. Use `dstack run get --json` for machine decisions. -- Use `dstack event --within-run ` and `dstack logs ` to understand provisioning, pulling, startup, and runtime failures. -- A run can be `running` before the in-server proxy can answer. Before probing the service URL, check `dstack run get --json` for a running job and HTTP probe state. If a probe has `success_streak > 0`, proceed to a real model request. Do not block on a `registered` field if it is absent or null. -- If a run is externally stopped or has `termination_reason: stopped_by_user`, do not resubmit it. -- Stop a bad candidate before replacing it: `dstack stop -y`. -- Retry only after changing the underlying hypothesis, YAML, command, image, hardware, or constraints. - -Service essentials: - -- The final endpoint-backed YAML must be `type: service`; choose a concise, unique run name that is useful for debugging. -- The final service YAML must include `model: ` so dstack exposes a model URL. -- Services usually need `port: 8000` and commands that start an OpenAI-compatible server. -- Common starting points are `vllm serve ` and `python -m sglang.launch_server --model-path --host 0.0.0.0 --port 8000`, but verify current docs and model-specific notes before trusting these defaults. -- Pass endpoint environment variables by name, for example `env: [HF_TOKEN]`; never write secret values into YAML or logs. -- Choose `resources` from evidence: model size, framework requirements, GPU memory, disk needed for weights/cache, and current dstack offers/fleets. -- Do not turn the first matching offer into the service requirements. Use offer previews to - verify that the derived requirements are provisionable. Keep `resources` as ranges or - minimums whenever that is enough for correctness. -- Do not pin `gpu.name`, `instance_types`, regions, CPU, memory, or disk from a preview - offer unless there is a model/framework requirement or a documented provisioning reason. - The actual instance that succeeds is recorded after provisioning, not guessed before it. -- For OpenAI-compatible verification, use `service.model.base_url` from `dstack run get --json` and POST to `/chat/completions` with the requested model. If the base URL is relative, prefix it with the default project URL from `~/.dstack/config.yml`. -- After verification, read `dstack run get --json` again and report actual - provisioned hardware from the latest running job. If dstack fell back from one preview - offer to another, the final report must name the hardware that actually ran. - -Use dstack plans as real constraints. If the endpoint supplied `max_price`, `spot_policy`, backend, region, instance type, fleet, or reuse constraints, every offer lookup, preview, and submitted experiment must honor them. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md b/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md deleted file mode 100644 index 527cf51ae5..0000000000 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/recipes_guide.md +++ /dev/null @@ -1,25 +0,0 @@ -# Recipe and Hardware Grounding - -Before choosing a deployment, gather enough evidence to explain why the service command and hardware should work. - -Primary sources to prefer: - -- vLLM docs and recipes: https://docs.vllm.ai/ and https://recipes.vllm.ai/models.json -- SGLang docs and cookbook: https://docs.sglang.ai/ -- Hugging Face model cards and framework notes, for example https://huggingface.co/docs/transformers/model_doc/qwen3 and https://huggingface.co/Qwen/Qwen3-0.6B -- dstack documentation and local CLI help. - -Supporting sources for advanced direction, not v1 requirements: - -- https://www.wafer.ai/blog/glm52-amd -- https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development/ -- https://github.com/dstackai/dstack/pull/3856 - -Recipe selection rubric: - -- Prefer an official framework recipe for the exact model. If none exists, use the model family recipe plus the model card. -- Check whether the model needs `trust_remote_code`, special tokenizer/chat template handling, quantization, tensor parallelism, or specific framework versions. -- Estimate VRAM, GPU count, disk, and any CPU/memory needs before looking at offers. For a tiny model, avoid overbuying unless the cheaper path is unstable. For larger models, do not guess; use model size, precision, quantization, KV-cache, and tensor-parallel evidence. -- Treat concrete offers as availability evidence, not as requirements. A selected offer may fail or dstack may provision a different matching offer; final hardware evidence must come from the verified run. -- When evidence is uncertain, use a bounded experiment instead of guessing. -- Record recipe/source URLs in the final report so a learned preset has provenance. diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md index 87cc52fa1f..440cd5469c 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md @@ -1,15 +1,43 @@ # Objective -You are the server-side endpoint deployment agent for dstack. Your job is to turn an endpoint request into a working, verified dstack service for the requested model. +You are the server-side endpoint deployment agent for dstack. Produce one dstack service that serves the requested model and prove it with a real model API request. -This is a real deployment investigation, not a YAML-generation task. Use the local workspace to keep notes, service/dev/task YAML files, command transcripts, backend observations, and evidence. Use the real `dstack` CLI and shell commands directly. Verify CLI flags with `dstack --help` when unsure. +Use the real `dstack` CLI and shell commands. Do not call hidden server APIs or helper functions such as `find_model_recipes`, `submit_service`, `get_run_status`, or `get_run_logs`. -Use the embedded dstack skill guidance as the source of truth for normal dstack CLI workflows. Use the embedded dstack-development skill guidance when a dev environment would reduce uncertainty around images, commands, model download, serving parameters, or hardware before submitting the final service. +Load and follow `/dstack` for CLI/config rules. Load and follow `/dstack-prototyping` for recipe research, hardware fit, experiments, debugging, and service finalization. -Do not invent dstack flags or YAML properties. Do not call hidden server APIs. Do not wait for custom helper functions such as `find_model_recipes`, `submit_service`, or `get_run_logs`; use normal files, shell commands, web sources, and the `dstack` CLI. +Use primary sources when recipe or hardware behavior is uncertain. Save source URLs in `sources.jsonl` and include the relevant ones in `final_report.json`. -You may use network research through the available web tools and shell commands. Prefer primary, current sources and preserve URLs in the final report. Treat model cards, official serving docs, recipe indexes, and successful command/log evidence as stronger than generic snippets. +Success requires a final dstack service run that answers a model API request for the requested model. Run status, service probes, and clean logs are evidence, not success. -The final endpoint may only be reported as successful after the final dstack service run is running, exposes the model endpoint, and has answered a real model request for the requested model. +Write user-facing progress to `progress.jsonl` as single-line JSON objects: -If verification succeeds, stop investigating and produce the final report immediately. Do not continue optimizing, benchmarking, or trying alternate hardware after a correct working service is proven. +```json +{"phase":"research","message":"Checking vLLM and SGLang support for Qwen/Qwen3-0.6B"} +``` + +Use progress only for milestones: research direction, experiment choice, candidate submitted, provisioning observation, verification result, cleanup, or terminal failure. Do not write YAML, command output, long tables, raw traces, or secrets to `progress.jsonl`. + +Record each dev environment, task, or service candidate in `candidates.jsonl`: + +```json +{"name":"qwen-test","type":"service","purpose":"final candidate","config_path":"qwen-test.dstack.yml","status":"submitted","run_id":null} +``` + +Update `status` when the candidate is ruled out, stopped, failed, or verified. Stop candidates you have ruled out unless they are needed for active debugging. + +Do not use the endpoint name itself as a candidate run name. Endpoint progress logs are keyed by endpoint name, and same-name service runs make debugging ambiguous. Use short attempt-style names for services, such as `-1`, `-2`, etc. + +Make each service YAML self-contained. If endpoint constraints map to service YAML fields, include them in the YAML; use CLI flags only as a checked override for preview/apply, not as the only record of the final service. + +Do not wait only for log text. During provisioning/startup, poll `dstack run get --json` and break on terminal statuses such as `failed` or `terminated`. Logs explain why a process failed; run JSON decides whether to keep waiting. + +Use normal service logs first. Do not use `dstack logs -d` for framework/application failures unless normal logs and run JSON are insufficient, because diagnostic logs may include runtime environment details that should not become endpoint trace material. + +On startup or resume, inspect `agent_state.json`, `candidates.jsonl`, `commands.jsonl`, `verification.json`, and `final_report.json` before submitting anything new. Reuse an existing candidate only if it can still be verified. + +On success, write `verification.json` with the request URL, request model, response evidence, and final run name/id. Then write `final_report.json` and return the structured final report. + +On failure, write `final_report.json` with the failure summary and the evidence needed for the next attempt. Then return the structured final report. + +Stop after one correct service is verified. Do not benchmark, tune autoscaling, optimize hardware, or attempt P/D disaggregation unless the requested model cannot serve without it. diff --git a/src/dstack/_internal/server/services/endpoints/planning.py b/src/dstack/_internal/server/services/endpoints/planning.py index e2926145e8..3f8d145ef8 100644 --- a/src/dstack/_internal/server/services/endpoints/planning.py +++ b/src/dstack/_internal/server/services/endpoints/planning.py @@ -75,7 +75,7 @@ async def find_preset_planning_result( endpoint_model = endpoint_configuration.model.lower() presets = [ preset - for preset in await preset_service.list_presets() + for preset in await preset_service.list_presets(project.name) if preset.model.lower() == endpoint_model ] if not presets: diff --git a/src/dstack/_internal/server/services/endpoints/presets.py b/src/dstack/_internal/server/services/endpoints/presets.py index 70ddc0cd31..55f7ce1c74 100644 --- a/src/dstack/_internal/server/services/endpoints/presets.py +++ b/src/dstack/_internal/server/services/endpoints/presets.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from copy import deepcopy from pathlib import Path -from typing import Any, Optional, Sequence +from typing import Any, NoReturn, Optional, Sequence import yaml from pydantic import ValidationError, parse_obj_as @@ -18,6 +18,12 @@ from dstack._internal.core.models.endpoint_presets import ( EndpointPreset as EndpointPresetSummary, ) +from dstack._internal.core.models.endpoint_presets import ( + EndpointPresetDetails, +) +from dstack._internal.core.models.endpoint_presets import ( + EndpointPresetReplicaSpecGroup as EndpointPresetReplicaSpecGroupSummary, +) from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.resources import CPUSpec, ResourcesSpec @@ -49,16 +55,21 @@ class EndpointPreset(CoreModel): class EndpointPresetService(ABC): @abstractmethod - async def list_presets(self) -> list[EndpointPreset]: + async def list_presets(self, project_name: str) -> list[EndpointPreset]: + pass + + @abstractmethod + async def get_preset(self, project_name: str, name: str) -> Optional[EndpointPreset]: pass @abstractmethod - async def delete_preset(self, name: str) -> None: + async def delete_preset(self, project_name: str, name: str) -> None: pass @abstractmethod async def save_preset( self, + project_name: str, preset: EndpointPreset, comments: Optional[Sequence[str]] = None, ) -> EndpointPreset: @@ -67,27 +78,32 @@ async def save_preset( class LocalDirEndpointPresetService(EndpointPresetService): - def __init__(self, presets_dir: Path = settings.ENDPOINT_PRESETS_DIR) -> None: - self._presets_dir = presets_dir + def __init__(self, projects_dir: Path = settings.SERVER_PROJECTS_DIR_PATH) -> None: + self._projects_dir = projects_dir - async def list_presets(self) -> list[EndpointPreset]: - return await run_async(self._list_presets) + async def list_presets(self, project_name: str) -> list[EndpointPreset]: + return await run_async(self._list_presets, project_name) - async def delete_preset(self, name: str) -> None: - return await run_async(self._delete_preset, name) + async def get_preset(self, project_name: str, name: str) -> Optional[EndpointPreset]: + return await run_async(self._get_preset, project_name, name) + + async def delete_preset(self, project_name: str, name: str) -> None: + return await run_async(self._delete_preset, project_name, name) async def save_preset( self, + project_name: str, preset: EndpointPreset, comments: Optional[Sequence[str]] = None, ) -> EndpointPreset: - return await run_async(self._save_preset, preset, comments or []) + return await run_async(self._save_preset, project_name, preset, comments or []) - def _list_presets(self) -> list[EndpointPreset]: - if not self._presets_dir.exists(): + def _list_presets(self, project_name: str) -> list[EndpointPreset]: + presets_dir = self._get_project_presets_dir(project_name) + if not presets_dir.exists(): return [] presets = [] - for path in sorted(self._presets_dir.iterdir()): + for path in sorted(presets_dir.iterdir()): if path.suffix not in [".yml", ".yaml"]: continue preset = self._load_preset(path) @@ -95,10 +111,22 @@ def _list_presets(self) -> list[EndpointPreset]: presets.append(preset) return presets - def _delete_preset(self, name: str) -> None: - if not self._presets_dir.exists(): + def _get_preset(self, project_name: str, name: str) -> Optional[EndpointPreset]: + presets_dir = self._get_project_presets_dir(project_name) + if not presets_dir.exists(): + return None + for path in presets_dir.iterdir(): + if path.suffix not in [".yml", ".yaml"]: + continue + if _get_preset_name_from_path(path) == name: + return self._load_preset(path) + return None + + def _delete_preset(self, project_name: str, name: str) -> None: + presets_dir = self._get_project_presets_dir(project_name) + if not presets_dir.exists(): raise FileNotFoundError(name) - for path in self._presets_dir.iterdir(): + for path in presets_dir.iterdir(): if path.suffix not in [".yml", ".yaml"]: continue if _get_preset_name_from_path(path) == name: @@ -136,18 +164,24 @@ def _load_preset(self, path: Path) -> Optional[EndpointPreset]: logger.warning("Skipping endpoint preset %s: %s", path, e) return None - def _save_preset(self, preset: EndpointPreset, comments: Sequence[str]) -> EndpointPreset: + def _save_preset( + self, + project_name: str, + preset: EndpointPreset, + comments: Sequence[str], + ) -> EndpointPreset: _validate_preset_before_save(preset) - self._presets_dir.mkdir(parents=True, exist_ok=True) + presets_dir = self._get_project_presets_dir(project_name) + presets_dir.mkdir(parents=True, exist_ok=True) data = _preset_to_data(preset) content = _format_preset_comments(comments) + yaml.safe_dump(data, sort_keys=False) base_name = _slugify_preset_name(preset.name) suffix = 0 while True: name = base_name if suffix == 0 else f"{base_name}-{suffix + 1}" - path = self._presets_dir / f"{name}.dstack.yml" + path = presets_dir / f"{name}.dstack.yml" try: - self._write_preset_file(path=path, content=content) + self._write_preset_file(presets_dir=presets_dir, path=path, content=content) break except FileExistsError: suffix += 1 @@ -156,8 +190,11 @@ def _save_preset(self, preset: EndpointPreset, comments: Sequence[str]) -> Endpo raise ValueError(f"saved endpoint preset {path} could not be loaded") return saved_preset - def _write_preset_file(self, path: Path, content: str) -> None: - tmp_path = self._presets_dir / f".{path.name}.{uuid.uuid4().hex}.tmp" + def _get_project_presets_dir(self, project_name: str) -> Path: + return self._projects_dir / project_name / "presets" + + def _write_preset_file(self, presets_dir: Path, path: Path, content: str) -> None: + tmp_path = presets_dir / f".{path.name}.{uuid.uuid4().hex}.tmp" tmp_path.write_text(content, encoding="utf-8") try: os.link(tmp_path, path) @@ -176,7 +213,22 @@ def endpoint_preset_to_api_model(preset: EndpointPreset) -> EndpointPresetSummar return EndpointPresetSummary( name=preset.name, model=preset.model, - replica_spec_groups=preset.replica_spec_groups, + replica_spec_groups=[ + EndpointPresetReplicaSpecGroupSummary.parse_obj(group.dict()) + for group in preset.replica_spec_groups + ], + ) + + +def endpoint_preset_to_api_details(preset: EndpointPreset) -> EndpointPresetDetails: + return EndpointPresetDetails( + name=preset.name, + model=preset.model, + replica_spec_groups=[ + EndpointPresetReplicaSpecGroupSummary.parse_obj(group.dict()) + for group in preset.replica_spec_groups + ], + service=preset.configuration, ) @@ -398,20 +450,21 @@ def _validate_replica_resources_are_exact(resources: ResourcesSpec) -> None: _raise_loose_replica_resources() if resources.disk is None or not _is_exact_range(resources.disk.size): _raise_loose_replica_resources() - if resources.gpu is None or not _is_exact_range(resources.gpu.count): + gpu = resources.gpu + if gpu is None or not _is_exact_range(gpu.count): _raise_loose_replica_resources() - gpu_count = resources.gpu.count.min + gpu_count = gpu.count.min if gpu_count == 0: return - if resources.gpu.name is None or len(resources.gpu.name) != 1: + if gpu.name is None or len(gpu.name) != 1: _raise_loose_replica_resources() - if resources.gpu.memory is None or not _is_exact_range(resources.gpu.memory): + if gpu.memory is None or not _is_exact_range(gpu.memory): _raise_loose_replica_resources() - if resources.gpu.compute_capability is not None: + if gpu.compute_capability is not None: _raise_loose_replica_resources() -def _raise_loose_replica_resources() -> None: +def _raise_loose_replica_resources() -> NoReturn: raise ValueError("preset tested_resources must use exact replica resources") diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 218c141580..b001a0d3ab 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -36,9 +36,7 @@ def _getenv_non_empty(name: str) -> str | None: SERVER_DATA_DIR_PATH = SERVER_DIR_PATH / "data" SERVER_DATA_DIR_PATH.mkdir(parents=True, exist_ok=True) -ENDPOINT_PRESETS_DIR = Path( - os.getenv("DSTACK_SERVER_ENDPOINT_PRESETS_DIR", SERVER_DATA_DIR_PATH / "endpoint_presets") -).resolve() +SERVER_PROJECTS_DIR_PATH = SERVER_DIR_PATH / "projects" AGENT_ANTHROPIC_API_KEY = _getenv_non_empty("DSTACK_AGENT_ANTHROPIC_API_KEY") AGENT_CLAUDE_PATH = _getenv_non_empty("DSTACK_AGENT_CLAUDE_PATH") diff --git a/src/dstack/api/server/_endpoint_presets.py b/src/dstack/api/server/_endpoint_presets.py index bc58c07444..09ff0b08c7 100644 --- a/src/dstack/api/server/_endpoint_presets.py +++ b/src/dstack/api/server/_endpoint_presets.py @@ -2,8 +2,11 @@ from pydantic import parse_obj_as -from dstack._internal.core.models.endpoint_presets import EndpointPreset -from dstack._internal.server.schemas.endpoint_presets import DeleteEndpointPresetsRequest +from dstack._internal.core.models.endpoint_presets import EndpointPreset, EndpointPresetDetails +from dstack._internal.server.schemas.endpoint_presets import ( + DeleteEndpointPresetsRequest, + GetEndpointPresetRequest, +) from dstack.api.server._group import APIClientGroup @@ -12,6 +15,14 @@ def list(self, project_name: str) -> List[EndpointPreset]: resp = self._request(f"/api/project/{project_name}/endpoints/presets/list") return parse_obj_as(List[EndpointPreset.__response__], resp.json()) + def get(self, project_name: str, name: str) -> EndpointPresetDetails: + body = GetEndpointPresetRequest(name=name) + resp = self._request( + f"/api/project/{project_name}/endpoints/presets/get", + body=body.json(), + ) + return parse_obj_as(EndpointPresetDetails.__response__, resp.json()) + def delete(self, project_name: str, names: List[str]) -> None: body = DeleteEndpointPresetsRequest(names=names) self._request( diff --git a/src/dstack/api/server/_endpoints.py b/src/dstack/api/server/_endpoints.py index cb8cb0c4f7..aaac8f439d 100644 --- a/src/dstack/api/server/_endpoints.py +++ b/src/dstack/api/server/_endpoints.py @@ -5,9 +5,9 @@ from dstack._internal.core.models.endpoints import Endpoint, EndpointConfiguration, EndpointPlan from dstack._internal.server.schemas.endpoints import ( CreateEndpointRequest, - DeleteEndpointsRequest, GetEndpointPlanRequest, GetEndpointRequest, + StopEndpointsRequest, ) from dstack.api.server._group import APIClientGroup @@ -44,6 +44,6 @@ def create( resp = self._request(f"/api/project/{project_name}/endpoints/create", body=body.json()) return parse_obj_as(Endpoint.__response__, resp.json()) - def delete(self, project_name: str, names: List[str]) -> None: - body = DeleteEndpointsRequest(names=names) - self._request(f"/api/project/{project_name}/endpoints/delete", body=body.json()) + def stop(self, project_name: str, names: List[str]) -> None: + body = StopEndpointsRequest(names=names) + self._request(f"/api/project/{project_name}/endpoints/stop", body=body.json()) diff --git a/src/tests/_internal/cli/commands/test_endpoint.py b/src/tests/_internal/cli/commands/test_endpoint.py new file mode 100644 index 0000000000..f886db79dc --- /dev/null +++ b/src/tests/_internal/cli/commands/test_endpoint.py @@ -0,0 +1,108 @@ +import argparse +import base64 +from datetime import datetime, timezone +from types import SimpleNamespace +from uuid import uuid4 + +from dstack._internal.cli.commands.endpoint import EndpointCommand +from dstack._internal.core.models.endpoints import Endpoint, EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.logs import JobSubmissionLogs, LogEvent, LogEventSource + + +def _get_endpoint(status: EndpointStatus = EndpointStatus.RUNNING) -> Endpoint: + return Endpoint( + id=uuid4(), + name="qwen-endpoint", + project_name="main", + user="test-user", + configuration=EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B"), + created_at=datetime.now(timezone.utc), + last_processed_at=datetime.now(timezone.utc), + status=status, + run_name="qwen-endpoint-1", + ) + + +class _FakeEndpoints: + def __init__(self, endpoint: Endpoint): + self._endpoint = endpoint + self.get_requests = [] + self.stopped_names = [] + + def get(self, project_name, name): + self.get_requests.append( + { + "project_name": project_name, + "name": name, + } + ) + return self._endpoint + + def stop(self, project_name, names): + self.stopped_names.extend(names) + + +class _FakeLogs: + def __init__(self): + self.requests = [] + + def poll(self, project_name, body): + self.requests.append(body) + return JobSubmissionLogs( + logs=[ + LogEvent( + timestamp=datetime.now(timezone.utc), + log_source=LogEventSource.STDOUT, + message=base64.b64encode(b"agent log\n").decode(), + ) + ], + ) + + +def _get_command(endpoint: Endpoint) -> EndpointCommand: + command = EndpointCommand.__new__(EndpointCommand) + command.api = SimpleNamespace( + project="main", + client=SimpleNamespace( + endpoints=_FakeEndpoints(endpoint), + logs=_FakeLogs(), + ), + ) + return command + + +class TestEndpointCommand: + def test_reads_endpoint_logs(self): + endpoint = _get_endpoint() + command = _get_command(endpoint) + + logs = list(command._get_endpoint_logs(endpoint=endpoint, start_time=None)) + + assert logs == [b"agent log\n"] + request = command.api.client.logs.requests[0] + assert request.run_name == endpoint.name + assert request.job_submission_id == endpoint.id + + def test_stop_endpoint(self): + endpoint = _get_endpoint() + command = _get_command(endpoint) + + command._stop(argparse.Namespace(name=endpoint.name, yes=True)) + + assert command.api.client.endpoints.stopped_names == [endpoint.name] + + def test_stop_already_stopped_endpoint_is_noop(self): + endpoint = _get_endpoint(status=EndpointStatus.STOPPED) + command = _get_command(endpoint) + + command._stop(argparse.Namespace(name=endpoint.name, yes=True)) + + assert command.api.client.endpoints.stopped_names == [] + + def test_stop_failed_endpoint_is_noop(self): + endpoint = _get_endpoint(status=EndpointStatus.FAILED) + command = _get_command(endpoint) + + command._stop(argparse.Namespace(name=endpoint.name, yes=True)) + + assert command.api.client.endpoints.stopped_names == [] diff --git a/src/tests/_internal/cli/commands/test_logs.py b/src/tests/_internal/cli/commands/test_logs.py index 53cf05d8a9..4600c91311 100644 --- a/src/tests/_internal/cli/commands/test_logs.py +++ b/src/tests/_internal/cli/commands/test_logs.py @@ -1,33 +1,15 @@ import argparse -import base64 -from datetime import datetime, timezone from types import SimpleNamespace -from uuid import uuid4 + +import pytest from dstack._internal.cli.commands.logs import LogsCommand -from dstack._internal.core.errors import ResourceNotExistsError -from dstack._internal.core.models.endpoints import Endpoint, EndpointConfiguration, EndpointStatus -from dstack._internal.core.models.logs import JobSubmissionLogs, LogEvent, LogEventSource - - -def _get_endpoint(run_name: str | None = None) -> Endpoint: - return Endpoint( - id=uuid4(), - name="qwen-endpoint", - project_name="main", - user="test-user", - configuration=EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B"), - created_at=datetime.now(timezone.utc), - last_processed_at=datetime.now(timezone.utc), - status=EndpointStatus.PROVISIONING, - deleted=False, - run_name=run_name, - ) +from dstack._internal.core.errors import CLIError class _FakeRun: def logs(self, **kwargs): - yield b"service log\n" + yield b"run log\n" class _FakeRuns: @@ -40,47 +22,15 @@ def get(self, name): return self._run -class _FakeEndpoints: - def __init__(self, endpoint): - self._endpoint = endpoint - - def get(self, project_name, name): - if self._endpoint is None: - raise ResourceNotExistsError() - return self._endpoint - - -class _FakeLogs: - def __init__(self): - self.requests = [] - - def poll(self, project_name, body): - self.requests.append(body) - return JobSubmissionLogs( - logs=[ - LogEvent( - timestamp=datetime.now(timezone.utc), - log_source=LogEventSource.STDOUT, - message=base64.b64encode(b"agent log\n").decode(), - ) - ], - ) - - -def _get_command(endpoint=None, run=None) -> LogsCommand: +def _get_command(run=None) -> LogsCommand: command = LogsCommand.__new__(LogsCommand) command.api = SimpleNamespace( - project="main", runs=_FakeRuns(run=run), - client=SimpleNamespace( - endpoints=_FakeEndpoints(endpoint=endpoint), - logs=_FakeLogs(), - ), ) return command -def _get_args(name="qwen-endpoint"): +def _get_args(name="qwen-run"): return argparse.Namespace( run_name=name, diagnose=False, @@ -90,35 +40,16 @@ def _get_args(name="qwen-endpoint"): class TestLogsCommand: - def test_reads_endpoint_agent_logs_when_backing_run_is_not_known(self): - endpoint = _get_endpoint() - command = _get_command(endpoint=endpoint) - - logs = list( - command._get_endpoint_logs(endpoint=endpoint, args=_get_args(), start_time=None) - ) - - assert logs == [b"agent log\n"] - request = command.api.client.logs.requests[0] - assert request.run_name == endpoint.name - assert request.job_submission_id == endpoint.id - - def test_reads_backing_service_logs_when_endpoint_has_run(self): - endpoint = _get_endpoint(run_name="qwen-service") - command = _get_command(endpoint=endpoint, run=_FakeRun()) + def test_reads_run_logs(self): + command = _get_command(run=_FakeRun()) - logs = list( - command._get_endpoint_logs(endpoint=endpoint, args=_get_args(), start_time=None) - ) - - assert logs == [b"service log\n"] - assert command.api.runs.requested_names == ["qwen-service"] + logs = list(command._get_logs(args=_get_args(), start_time=None)) - def test_endpoint_name_takes_precedence_over_same_name_run(self): - endpoint = _get_endpoint() - command = _get_command(endpoint=endpoint, run=_FakeRun()) + assert logs == [b"run log\n"] + assert command.api.runs.requested_names == ["qwen-run"] - logs = list(command._get_logs(args=_get_args(), start_time=None)) + def test_errors_when_run_is_not_found(self): + command = _get_command(run=None) - assert logs == [b"agent log\n"] - assert command.api.runs.requested_names == [] + with pytest.raises(CLIError, match="Run qwen-run not found"): + list(command._get_logs(args=_get_args(), start_time=None)) diff --git a/src/tests/_internal/cli/services/configurators/test_endpoint.py b/src/tests/_internal/cli/services/configurators/test_endpoint.py index ae40fa6c8b..2e0d34560f 100644 --- a/src/tests/_internal/cli/services/configurators/test_endpoint.py +++ b/src/tests/_internal/cli/services/configurators/test_endpoint.py @@ -6,23 +6,28 @@ import dstack._internal.cli.services.configurators.endpoint as endpoint_configurator_module from dstack._internal.cli.services.configurators.endpoint import ( + EndpointConfigurator, _get_apply_confirm_message, _make_endpoint_url_absolute, _print_endpoint_plan, _print_finished_endpoint_message, _print_submitted_endpoint_message, ) +from dstack._internal.core.errors import ConfigurationError from dstack._internal.core.models.common import ApplyAction from dstack._internal.core.models.endpoints import ( Endpoint, EndpointConfiguration, EndpointPlan, + EndpointPlanJobOffers, + EndpointPlanReplicaSpecGroup, EndpointPresetPolicy, EndpointProvisioningPlanAgent, EndpointProvisioningPlanNone, EndpointProvisioningPlanPreset, EndpointStatus, ) +from dstack._internal.core.models.resources import ResourcesSpec def _get_endpoint_plan( @@ -68,7 +73,6 @@ def _get_current_endpoint() -> Endpoint: created_at=datetime.now(timezone.utc), last_processed_at=datetime.now(timezone.utc), status=EndpointStatus.RUNNING, - deleted=False, ) @@ -99,8 +103,8 @@ def test_prints_stable_properties_for_missing_provisioning_path( output = console.export_text() assert "Project main" in output - assert "Endpoint qwen-endpoint" in output - assert "Resources -" in output + assert "Endpoint" not in output + assert "Resources" not in output assert "Spot policy auto" in output assert "Max price off" in output assert "Preset policy reuse-or-create" in output @@ -110,6 +114,40 @@ def test_prints_stable_properties_for_missing_provisioning_path( assert "Action" not in output assert "Provisioning" not in output + def test_prints_preset_without_resources_property(self, monkeypatch: pytest.MonkeyPatch): + console = _patch_console(monkeypatch) + resources = ResourcesSpec.parse_obj({"gpu": "16GB", "disk": "60GB"}) + plan = _get_endpoint_plan( + EndpointProvisioningPlanPreset( + preset_name="qwen-vllm", + service_name="qwen-endpoint-serving", + replica_spec_groups=[ + EndpointPlanReplicaSpecGroup( + name="0", + resources=resources, + tested_resources=[resources], + ) + ], + job_offers=[ + EndpointPlanJobOffers( + replica_group="0", + resources=resources, + spot=False, + max_price=0.5, + offers=[], + total_offers=0, + max_offer_price=None, + ) + ], + ) + ) + + _print_endpoint_plan(plan) + + output = console.export_text() + assert "Resources" not in output + assert "Preset qwen-vllm" in output + def test_prints_agent_reason_when_preset_has_no_offers(self, monkeypatch: pytest.MonkeyPatch): console = _patch_console(monkeypatch) plan = _get_endpoint_plan( @@ -212,3 +250,12 @@ def test_asks_to_override_same_name_run(self): _get_apply_confirm_message(plan, stop_run_name="qwen-endpoint") == "Stop and override the run [code]qwen-endpoint[/]?" ) + + +class TestEndpointConfigurator: + def test_delete_configuration_is_not_supported(self): + configurator = EndpointConfigurator.__new__(EndpointConfigurator) + conf = EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B") + + with pytest.raises(ConfigurationError, match="dstack endpoint stop"): + configurator.delete_configuration(conf, "endpoint.dstack.yml", command_args=None) diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py index 3906367a72..b0d8a919fe 100644 --- a/src/tests/_internal/cli/utils/test_endpoint.py +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -31,7 +31,6 @@ def _get_endpoint( last_processed_at=created_at, status=status, status_message=status_message, - deleted=False, ) @@ -106,11 +105,23 @@ def test_running_status_is_colored(self): status_column = next(column for column in table.columns if column.header == "STATUS") assert status_column._cells == ["[bold sea_green3]running[/]"] - def test_agenting_status_is_colored(self): - table = get_endpoints_table([_get_endpoint(status=EndpointStatus.AGENTING)]) + def test_clauding_status_is_colored(self): + table = get_endpoints_table([_get_endpoint(status=EndpointStatus.CLAUDING)]) status_column = next(column for column in table.columns if column.header == "STATUS") - assert status_column._cells == ["[bold medium_purple1]agenting[/]"] + assert status_column._cells == ["[bold medium_purple1]clauding[/]"] + + def test_stopping_status_is_colored(self): + table = get_endpoints_table([_get_endpoint(status=EndpointStatus.STOPPING)]) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[bold deep_sky_blue1]stopping[/]"] + + def test_stopped_status_is_colored(self): + table = get_endpoints_table([_get_endpoint(status=EndpointStatus.STOPPED)]) + + status_column = next(column for column in table.columns if column.header == "STATUS") + assert status_column._cells == ["[grey62]stopped[/]"] class TestGetEndpointTable: @@ -164,8 +175,8 @@ def test_default_shows_unfinished_and_latest_finished(self): created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), ), _get_endpoint( - name="agenting", - status=EndpointStatus.AGENTING, + name="clauding", + status=EndpointStatus.CLAUDING, created_at=datetime(2026, 1, 4, tzinfo=timezone.utc), ), ] @@ -173,7 +184,7 @@ def test_default_shows_unfinished_and_latest_finished(self): filtered = filter_endpoints_for_listing(endpoints) assert [endpoint.name for endpoint in filtered] == [ - "agenting", + "clauding", "failed-new", "running", ] @@ -204,15 +215,15 @@ def test_default_shows_latest_finished_and_unfinished_in_watch(self): created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), ), _get_endpoint( - name="agenting", - status=EndpointStatus.AGENTING, + name="clauding", + status=EndpointStatus.CLAUDING, created_at=datetime(2026, 1, 4, tzinfo=timezone.utc), ), ] filtered = filter_endpoints_for_listing(endpoints) - assert [endpoint.name for endpoint in filtered] == ["agenting", "failed-new"] + assert [endpoint.name for endpoint in filtered] == ["clauding", "failed-new"] def test_all_shows_all_sorted_newest_first(self): endpoints = [ diff --git a/src/tests/_internal/cli/utils/test_preset.py b/src/tests/_internal/cli/utils/test_preset.py index b2c0b1baa4..2fe75ebc64 100644 --- a/src/tests/_internal/cli/utils/test_preset.py +++ b/src/tests/_internal/cli/utils/test_preset.py @@ -33,10 +33,10 @@ def test_shows_single_group_preset(self): assert table.columns[0]._cells == ["qwen-qwen3-0-6b"] assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B"] assert table.columns[2]._cells == [ - "cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1", + "cpu=2.. mem=8GB.. disk=100GB.. gpu=16GB:1..", ] - def test_shows_individual_replicas_without_group_for_implicit_group(self): + def test_shows_single_implicit_group_once_even_with_multiple_tested_replicas(self): preset = EndpointPreset( name="qwen", model="Qwen/Qwen3-0.6B", @@ -66,12 +66,10 @@ def test_shows_individual_replicas_without_group_for_implicit_group(self): table = get_endpoint_presets_table([preset]) - assert table.columns[0]._cells == ["qwen", " replica=0", " replica=1"] - assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B", "", ""] + assert table.columns[0]._cells == ["qwen"] + assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B"] assert table.columns[2]._cells == [ - "", - "cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1", - "cpu=9 mem=50GB disk=60GB gpu=A40:48GB:1", + "cpu=2.. mem=8GB.. disk=100GB.. gpu=16GB:1..", ] def test_shows_grouped_replica_specs(self): @@ -115,14 +113,12 @@ def test_shows_grouped_replica_specs(self): assert table.columns[0]._cells == [ "qwen-pd", - " group=router replica=0", - " group=worker replica=0", - " replica=1", + " group=router", + " group=worker", ] - assert table.columns[1]._cells == ["Qwen/Qwen3-30B-A3B", "", "", ""] + assert table.columns[1]._cells == ["Qwen/Qwen3-30B-A3B", "", ""] assert table.columns[2]._cells == [ "", - "cpu=8 mem=16GB disk=100GB", - "cpu=14 mem=64GB disk=200GB gpu=L4:24GB:1", - "cpu=14 mem=64GB disk=200GB gpu=L4:24GB:1", + "cpu=4 mem=8GB.. disk=100GB..", + "cpu=2.. mem=8GB.. disk=100GB.. gpu=24GB:1..", ] diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index 568c7968b7..cc75fe473a 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -73,7 +73,6 @@ def _endpoint_to_pipeline_item(endpoint_model: EndpointModel) -> EndpointPipelin lock_expires_at=endpoint_model.lock_expires_at, prev_lock_expired=False, status=endpoint_model.status, - to_be_deleted=endpoint_model.to_be_deleted, ) @@ -86,7 +85,6 @@ async def _lock_endpoint_model(session: AsyncSession, endpoint_model: EndpointMo async def _create_endpoint_model( session: AsyncSession, status: EndpointStatus = EndpointStatus.SUBMITTED, - to_be_deleted: bool = False, user_ssh_public_key: str | None = None, ) -> EndpointModel: project = await create_project(session=session) @@ -104,7 +102,6 @@ async def _create_endpoint_model( configuration=configuration.json(), created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), last_processed_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), - to_be_deleted=to_be_deleted, ) endpoint_model.lock_token = uuid.uuid4() endpoint_model.lock_expires_at = datetime(2025, 1, 2, 3, 4, tzinfo=timezone.utc) @@ -244,6 +241,28 @@ def _get_agent_run_name(suffix: str = "candidate") -> str: return f"agent-{suffix}" +def _make_preset_plan(run_name: str = "qwen-endpoint-serving") -> Mock: + preset_plan = Mock() + preset_plan.preset.name = "qwen" + preset_plan.run_plan.run_spec = RunSpec( + run_name=run_name, + configuration=ServiceConfiguration.parse_obj( + { + "type": "service", + "name": run_name, + "commands": [ + "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-0.6B", + "resources": {"gpu": "16GB"}, + } + ), + ) + preset_plan.run_plan.current_resource = None + return preset_plan + + async def _get_endpoint_run_submissions( session: AsyncSession, endpoint_model: EndpointModel, @@ -360,7 +379,7 @@ async def test_rejects_run_already_recorded_for_another_endpoint( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) class TestEndpointWorkerSubmitted: - async def test_fails_when_same_name_run_exists_without_link( + async def test_fails_when_preset_run_name_exists_without_link( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) @@ -375,7 +394,9 @@ async def test_fails_when_same_name_run_exists_without_link( patch( "dstack._internal.server.background.pipeline_tasks.endpoints." "find_preset_planning_result", - new=AsyncMock(), + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=_make_preset_plan()) + ), ) as find_preset_planning_result_mock, patch( "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", @@ -393,7 +414,7 @@ async def test_fails_when_same_name_run_exists_without_link( assert endpoint_model.service_run_id is None assert run.status == RunStatus.PROVISIONING assert run.deleted is False - find_preset_planning_result_mock.assert_not_awaited() + find_preset_planning_result_mock.assert_awaited_once() apply_plan_mock.assert_not_awaited() events = await list_events(session) assert len(events) == 1 @@ -402,7 +423,7 @@ async def test_fails_when_same_name_run_exists_without_link( "(Run name 'qwen-endpoint-serving' is taken by an existing run)" ) - async def test_fails_when_run_name_is_taken_by_another_user( + async def test_fails_when_preset_run_name_is_taken_by_another_user( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) @@ -421,7 +442,9 @@ async def test_fails_when_run_name_is_taken_by_another_user( patch( "dstack._internal.server.background.pipeline_tasks.endpoints." "find_preset_planning_result", - new=AsyncMock(), + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=_make_preset_plan()) + ), ) as find_preset_planning_result_mock, patch( "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", @@ -439,7 +462,7 @@ async def test_fails_when_run_name_is_taken_by_another_user( assert endpoint_model.service_run_id is None assert run.status == RunStatus.RUNNING assert run.deleted is False - find_preset_planning_result_mock.assert_not_awaited() + find_preset_planning_result_mock.assert_awaited_once() apply_plan_mock.assert_not_awaited() events = await list_events(session) assert len(events) == 1 @@ -448,7 +471,7 @@ async def test_fails_when_run_name_is_taken_by_another_user( "(Run name 'qwen-endpoint-serving' is taken by an existing run)" ) - async def test_fails_when_run_name_is_taken_by_pre_existing_run( + async def test_fails_when_preset_run_name_is_taken_by_pre_existing_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) @@ -466,7 +489,9 @@ async def test_fails_when_run_name_is_taken_by_pre_existing_run( with patch( "dstack._internal.server.background.pipeline_tasks.endpoints." "find_preset_planning_result", - new=AsyncMock(), + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=_make_preset_plan()) + ), ) as find_preset_planning_result_mock: await worker.process(_endpoint_to_pipeline_item(endpoint_model)) @@ -479,7 +504,43 @@ async def test_fails_when_run_name_is_taken_by_pre_existing_run( assert endpoint_model.service_run_id is None assert run.status == RunStatus.RUNNING assert run.deleted is False - find_preset_planning_result_mock.assert_not_awaited() + find_preset_planning_result_mock.assert_awaited_once() + + async def test_preset_run_name_conflict_does_not_block_clauding_without_preset( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.RUNNING, + link_endpoint=False, + ) + agent_service = _FakeAgentService() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ) as find_preset_planning_result_mock, + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status_message is None + assert endpoint_model.service_run_id is None + assert endpoint_model.provisioning_method == "agent" + assert run.status == RunStatus.RUNNING + assert run.deleted is False + find_preset_planning_result_mock.assert_awaited_once() + agent_service.provision_endpoint.assert_not_awaited() async def test_finished_same_name_run_does_not_block_preset_submission( self, test_db, session: AsyncSession, worker: EndpointWorker @@ -566,7 +627,7 @@ async def test_submitted_to_failed_without_provisioning_path( "DSTACK_AGENT_ANTHROPIC_API_KEY is not set.)" ) - async def test_submitted_to_agenting_with_agent( + async def test_submitted_to_clauding_with_agent( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) @@ -586,14 +647,14 @@ async def test_submitted_to_agenting_with_agent( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.AGENTING + assert endpoint_model.status == EndpointStatus.CLAUDING assert endpoint_model.status_message is None assert endpoint_model.service_run_id is None assert endpoint_model.provisioning_method == "agent" agent_service.provision_endpoint.assert_not_awaited() events = await list_events(session) assert len(events) == 1 - assert events[0].message == "Endpoint status changed SUBMITTED -> AGENTING" + assert events[0].message == "Endpoint status changed SUBMITTED -> CLAUDING" async def test_submitted_to_provisioning_with_matching_preset( self, test_db, session: AsyncSession, worker: EndpointWorker @@ -698,65 +759,32 @@ async def test_submitted_to_failed_when_preset_submission_fails( ), patch( "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", - new=AsyncMock(side_effect=ServerClientError("Cannot override active run")), + new=AsyncMock(side_effect=ServerClientError("Cannot override running run")), ) as apply_plan_mock, ): await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) assert endpoint_model.status == EndpointStatus.FAILED - assert endpoint_model.status_message == "Cannot override active run" + assert endpoint_model.status_message == "Cannot override running run" apply_plan_mock.assert_awaited_once() events = await list_events(session) assert len(events) == 1 assert ( events[0].message - == "Endpoint status changed SUBMITTED -> FAILED (Cannot override active run)" - ) - - async def test_delete_request_after_fetch_prevents_preset_submission( - self, test_db, session: AsyncSession, worker: EndpointWorker - ): - endpoint_model = await _create_endpoint_model( - session=session, - user_ssh_public_key="ssh-rsa test", + == "Endpoint status changed SUBMITTED -> FAILED (Cannot override running run)" ) - item = _endpoint_to_pipeline_item(endpoint_model) - endpoint_model.to_be_deleted = True - await session.commit() - - with ( - patch( - "dstack._internal.server.background.pipeline_tasks.endpoints." - "find_preset_planning_result", - new=AsyncMock(), - ) as find_preset_planning_result_mock, - patch( - "dstack._internal.server.background.pipeline_tasks.endpoints.runs_services.apply_plan", - new=AsyncMock(), - ) as apply_plan_mock, - ): - await worker.process(item) - - await session.refresh(endpoint_model) - assert endpoint_model.deleted is True - assert endpoint_model.deleted_at is not None - find_preset_planning_result_mock.assert_not_awaited() - apply_plan_mock.assert_not_awaited() - events = await list_events(session) - assert len(events) == 1 - assert events[0].message == "Endpoint deleted" @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) class TestEndpointWorkerProvisioning: - async def test_agenting_does_not_fail_on_legacy_same_name_run_conflict( + async def test_clauding_does_not_fail_on_legacy_same_name_run_conflict( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) legacy_run = await _create_backing_service_run( session=session, @@ -772,7 +800,9 @@ async def test_agenting_does_not_fail_on_legacy_same_name_run_conflict( ) agent_service = _FakeAgentService(result=_get_verified_agent_result(agent_run)) preset_service = Mock() - preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) with ( patch( @@ -798,14 +828,14 @@ async def test_agenting_does_not_fail_on_legacy_same_name_run_conflict( assert agent_run.deleted is False events = await list_events(session) assert len(events) == 1 - assert events[0].message == "Endpoint status changed AGENTING -> RUNNING" + assert events[0].message == "Endpoint status changed CLAUDING -> RUNNING" async def test_fails_when_agent_reports_foreign_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) repo = await create_repo(session=session, project_id=endpoint_model.project_id) another_user = await create_user(session=session, name="another-user") @@ -837,7 +867,7 @@ async def test_fails_when_agent_reports_foreign_run( events = await list_events(session) assert len(events) == 1 assert ( - events[0].message == "Endpoint status changed AGENTING -> FAILED " + events[0].message == "Endpoint status changed CLAUDING -> FAILED " "(Run 'agent-candidate' is not owned by the endpoint user)" ) @@ -846,7 +876,9 @@ async def test_agent_creates_ready_service_and_endpoint_becomes_running( ): endpoint_model = await _create_endpoint_model(session=session) preset_service = Mock() - preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) async def provision_endpoint(endpoint_model, pipeline_hinter): return await _create_ready_backing_service_run_for_agent(endpoint_model.id) @@ -871,7 +903,7 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): ): await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.AGENTING + assert endpoint_model.status == EndpointStatus.CLAUDING await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) await worker.process(_endpoint_to_pipeline_item(endpoint_model)) @@ -895,7 +927,7 @@ async def test_agent_reported_run_id_links_backing_service_run( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -928,7 +960,9 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): agent_service = _FakeAgentService(side_effect=provision_endpoint) preset_service = Mock() - preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) with ( patch( @@ -954,12 +988,12 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): assert submissions[0].run_id == endpoint_model.service_run_id preset_service.save_preset.assert_awaited_once() - async def test_agenting_linked_run_waits_without_showing_provisioning( + async def test_clauding_linked_run_waits_without_showing_provisioning( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await _create_backing_service_run( @@ -971,7 +1005,7 @@ async def test_agenting_linked_run_waits_without_showing_provisioning( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.AGENTING + assert endpoint_model.status == EndpointStatus.CLAUDING assert endpoint_model.status_message is None events = await list_events(session) assert events == [] @@ -981,7 +1015,7 @@ async def test_agent_reported_run_without_verification_report_fails_endpoint( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1020,7 +1054,7 @@ async def test_agent_failure_fails_endpoint( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1039,12 +1073,35 @@ async def test_agent_failure_fails_endpoint( assert endpoint_model.status_message == "agent could not find a deployable recipe" agent_service.provision_endpoint.assert_awaited_once() + async def test_agent_in_progress_keeps_endpoint_clauding( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.CLAUDING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + agent_service = _FakeAgentService(result=AgentProvisioningResult(in_progress=True)) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status_message is None + assert endpoint_model.service_run_id is None + agent_service.provision_endpoint.assert_awaited_once() + async def test_agent_error_status_message_is_compact( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1072,7 +1129,7 @@ async def test_agent_failure_report_status_message_is_compact( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1107,7 +1164,7 @@ async def test_agent_failure_does_not_stop_unlinked_same_name_run( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.AGENTING, + status=EndpointStatus.CLAUDING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1198,7 +1255,9 @@ async def test_saves_preset_when_agent_backing_service_becomes_running( endpoint_model.provisioning_method = "agent" await _create_backing_service_run(session=session, endpoint_model=endpoint_model) preset_service = Mock() - preset_service.save_preset = AsyncMock(side_effect=lambda preset, comments: preset) + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) with patch( "dstack._internal.server.background.pipeline_tasks.endpoints." @@ -1210,13 +1269,14 @@ async def test_saves_preset_when_agent_backing_service_becomes_running( await session.refresh(endpoint_model) assert endpoint_model.status == EndpointStatus.RUNNING preset_service.save_preset.assert_awaited_once() - saved_preset = preset_service.save_preset.await_args.args[0] + assert preset_service.save_preset.await_args.args[0] == "test_project" + saved_preset = preset_service.save_preset.await_args.args[1] assert saved_preset.model == "Qwen/Qwen3-0.6B" assert [group.name for group in saved_preset.replica_spec_groups] == ["0"] comments = preset_service.save_preset.await_args.kwargs["comments"] assert f"endpoint: {endpoint_model.name}" in comments - async def test_preset_save_failure_does_not_block_agent_endpoint_activation( + async def test_preset_save_failure_does_not_block_agent_endpoint_running( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( @@ -1321,8 +1381,8 @@ async def test_stops_running_backing_run_when_endpoint_fails( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) -class TestEndpointWorkerActive: - async def test_ready_backing_service_keeps_endpoint_active( +class TestEndpointWorkerRunning: + async def test_ready_backing_service_keeps_endpoint_running( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( @@ -1340,7 +1400,7 @@ async def test_ready_backing_service_keeps_endpoint_active( events = await list_events(session) assert events == [] - async def test_not_ready_backing_service_keeps_endpoint_active( + async def test_not_ready_backing_service_keeps_endpoint_running( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( @@ -1422,32 +1482,29 @@ async def test_stops_running_backing_run_when_endpoint_fails( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) -class TestEndpointWorkerDeleted: - async def test_marks_endpoint_deleted( +class TestEndpointWorkerStopping: + async def test_marks_endpoint_stopped_without_backing_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.FAILED, - to_be_deleted=True, + status=EndpointStatus.STOPPING, ) await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.deleted is True - assert endpoint_model.deleted_at is not None + assert endpoint_model.status == EndpointStatus.STOPPED + assert endpoint_model.status_message is None events = await list_events(session) - assert len(events) == 1 - assert events[0].message == "Endpoint deleted" + assert events[0].message == "Endpoint status changed STOPPING -> STOPPED" - async def test_stops_active_backing_run_before_deleting_endpoint( + async def test_stops_backing_run_before_stopping_endpoint( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.RUNNING, - to_be_deleted=True, + status=EndpointStatus.STOPPING, ) run = await _create_backing_service_run( session=session, @@ -1459,43 +1516,16 @@ async def test_stops_active_backing_run_before_deleting_endpoint( await session.refresh(endpoint_model) await session.refresh(run) - assert endpoint_model.deleted is False - assert endpoint_model.deleted_at is None + assert endpoint_model.status == EndpointStatus.STOPPING assert run.status == RunStatus.TERMINATING assert run.deleted is False - async def test_deletes_endpoint_without_stopping_unlinked_same_name_run( + async def test_marks_endpoint_stopped_after_backing_run_finishes( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.RUNNING, - to_be_deleted=True, - ) - run = await _create_backing_service_run( - session=session, - endpoint_model=endpoint_model, - status=RunStatus.RUNNING, - link_endpoint=False, - ) - - await worker.process(_endpoint_to_pipeline_item(endpoint_model)) - - await session.refresh(endpoint_model) - await session.refresh(run) - assert endpoint_model.deleted is True - assert endpoint_model.deleted_at is not None - assert endpoint_model.service_run_id is None - assert run.status == RunStatus.RUNNING - assert run.deleted is False - - async def test_deletes_finished_backing_run_before_deleting_endpoint( - self, test_db, session: AsyncSession, worker: EndpointWorker - ): - endpoint_model = await _create_endpoint_model( - session=session, - status=EndpointStatus.RUNNING, - to_be_deleted=True, + status=EndpointStatus.STOPPING, ) run = await _create_backing_service_run( session=session, @@ -1507,11 +1537,7 @@ async def test_deletes_finished_backing_run_before_deleting_endpoint( await session.refresh(endpoint_model) await session.refresh(run) - assert endpoint_model.deleted is True - assert endpoint_model.deleted_at is not None - assert run.deleted is True + assert endpoint_model.status == EndpointStatus.STOPPED + assert run.deleted is False events = await list_events(session) - assert [event.message for event in events] == [ - "Run deleted", - "Endpoint deleted", - ] + assert events[0].message == "Endpoint status changed STOPPING -> STOPPED" diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index de0ee22701..d7e209f0ba 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -262,8 +262,6 @@ async def test_creates_endpoint(self, test_db, session: AsyncSession, client: As assert body["last_processed_at"] == "2023-01-02T03:04:00+00:00" assert body["status"] == "submitted" assert body["status_message"] is None - assert body["deleted"] is False - assert body["deleted_at"] is None assert body["run_name"] is None assert body["url"] is None assert body["error"] is None @@ -276,7 +274,7 @@ async def test_creates_endpoint(self, test_db, session: AsyncSession, client: As @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - async def test_recreates_terminal_endpoint( + async def test_resubmits_terminal_endpoint( self, test_db, session: AsyncSession, client: AsyncClient ): user = await create_user(session, global_role=GlobalRole.USER) @@ -290,6 +288,8 @@ async def test_recreates_terminal_endpoint( user=user, status=EndpointStatus.FAILED, ) + previous_endpoint.status_message = "old failure" + await session.commit() response = await client.post( f"/api/project/{project.name}/endpoints/create", @@ -304,16 +304,15 @@ async def test_recreates_terminal_endpoint( ) assert response.status_code == 200, response.json() - new_endpoint_id = UUID(response.json()["id"]) - assert new_endpoint_id != previous_endpoint.id + endpoint_id = UUID(response.json()["id"]) + assert endpoint_id == previous_endpoint.id await session.refresh(previous_endpoint) - assert previous_endpoint.deleted is True - assert previous_endpoint.deleted_at is not None - new_endpoint = await session.get(EndpointModel, new_endpoint_id) - assert new_endpoint is not None - assert new_endpoint.name == previous_endpoint.name - assert new_endpoint.status == EndpointStatus.SUBMITTED - assert new_endpoint.deleted is False + assert previous_endpoint.status == EndpointStatus.SUBMITTED + assert previous_endpoint.status_message is None + assert previous_endpoint.service_run_id is None + assert previous_endpoint.configuration != "" + res = await session.execute(select(EndpointModel)) + assert len(res.scalars().all()) == 1 @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -382,15 +381,15 @@ async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): assert response.status_code in [401, 403] -class TestDeleteEndpoint: +class TestStopEndpoint: @pytest.mark.asyncio async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): - response = await client.post("/api/project/main/endpoints/delete") + response = await client.post("/api/project/main/endpoints/stop") assert response.status_code in [401, 403] @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - async def test_marks_endpoint_for_deletion( + async def test_marks_endpoint_for_stopping( self, test_db, session: AsyncSession, client: AsyncClient ): user = await create_user(session, global_role=GlobalRole.USER) @@ -398,45 +397,53 @@ async def test_marks_endpoint_for_deletion( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) - create_response = await client.post( - f"/api/project/{project.name}/endpoints/create", - headers=get_auth_headers(user.token), - json={ - "configuration": { - "type": "endpoint", - "name": "qwen-endpoint", - "model": "Qwen/Qwen3-0.6B", - "env": {"HF_TOKEN": "secret"}, - } - }, + endpoint_model = await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.RUNNING, ) - assert create_response.status_code == 200, create_response.json() response = await client.post( - f"/api/project/{project.name}/endpoints/delete", + f"/api/project/{project.name}/endpoints/stop", headers=get_auth_headers(user.token), - json={"names": ["qwen-endpoint"]}, + json={"names": [endpoint_model.name]}, ) assert response.status_code == 200, response.json() - res = await session.execute(select(EndpointModel)) - endpoint_model = res.scalar_one() - assert endpoint_model.to_be_deleted - assert endpoint_model.deleted is False - assert endpoint_model.deleted_at is None + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.STOPPING + events = await list_events(session) + assert [e.message for e in events] == ["Endpoint marked for stopping"] - get_response = await client.post( - f"/api/project/{project.name}/endpoints/get", + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_does_not_stop_finished_endpoint( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + endpoint_model = await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.FAILED, + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/stop", headers=get_auth_headers(user.token), - json={"name": "qwen-endpoint"}, + json={"names": [endpoint_model.name]}, ) - assert get_response.status_code == 200 + assert response.status_code == 200, response.json() + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED events = await list_events(session) - assert [e.message for e in events] == [ - "Endpoint created. Status: SUBMITTED", - "Endpoint marked for deletion", - ] + assert events == [] class TestEndpointPresets: @@ -450,6 +457,11 @@ async def test_delete_returns_40x_if_not_authenticated(self, client: AsyncClient response = await client.post("/api/project/main/endpoints/presets/delete") assert response.status_code in [401, 403] + @pytest.mark.asyncio + async def test_get_returns_40x_if_not_authenticated(self, client: AsyncClient): + response = await client.post("/api/project/main/endpoints/presets/get") + assert response.status_code in [401, 403] + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_lists_endpoint_presets( @@ -483,6 +495,7 @@ async def test_lists_endpoint_presets( assert body[0]["replica_spec_groups"][0]["name"] == "0" assert body[0]["replica_spec_groups"][0]["resources"]["gpu"] is not None assert body[0]["replica_spec_groups"][0]["tested_resources"][0]["gpu"] is not None + assert preset_service.list_project_names == [project.name] @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @@ -512,19 +525,66 @@ async def test_deletes_endpoint_preset( assert response.status_code == 200, response.json() assert preset_service.deleted_names == ["qwen"] + assert preset_service.delete_project_names == [project.name] + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_gets_endpoint_preset( + self, + test_db, + session: AsyncSession, + client: AsyncClient, + monkeypatch: pytest.MonkeyPatch, + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + preset_service = _FakeEndpointPresetService([_endpoint_preset_plan().preset]) + monkeypatch.setattr( + "dstack._internal.server.routers.endpoints.get_endpoint_preset_service", + lambda: preset_service, + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/presets/get", + headers=get_auth_headers(user.token), + json={"name": "qwen"}, + ) + + assert response.status_code == 200, response.json() + body = response.json() + assert body["name"] == "qwen" + assert body["model"] == "Qwen/Qwen3-0.6B" + assert body["service"]["type"] == "service" + assert body["service"]["name"] == "qwen-endpoint-serving" + assert preset_service.get_project_names == [project.name] class _FakeEndpointPresetService: def __init__(self, presets): self._presets = presets + self.list_project_names = [] + self.get_project_names = [] + self.delete_project_names = [] self.deleted_names = [] - async def list_presets(self): + async def list_presets(self, project_name): + self.list_project_names.append(project_name) return self._presets - async def delete_preset(self, name): + async def get_preset(self, project_name, name): + self.get_project_names.append(project_name) + for preset in self._presets: + if preset.name == name: + return preset + return None + + async def delete_preset(self, project_name, name): if name not in {preset.name for preset in self._presets}: raise FileNotFoundError(name) + self.delete_project_names.append(project_name) self.deleted_names.append(name) diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index 41e81b9a69..c583c4bf39 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -8,6 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession import dstack._internal.server.services.endpoints.agent.claude as claude_module +from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus from dstack._internal.core.models.envs import Env from dstack._internal.core.models.profiles import SpotPolicy @@ -32,6 +33,8 @@ async def _create_endpoint_model( max_agent_budget: float | None = None, max_price: float | None = None, spot_policy: SpotPolicy | None = None, + backends: list[BackendType] | None = None, + fleets: list[str] | None = None, ) -> EndpointModel: user = await create_user(session=session, name="admin", token="user-token") project = await create_project(session=session, owner=user, name="main") @@ -42,6 +45,8 @@ async def _create_endpoint_model( max_agent_budget=max_agent_budget, max_price=max_price, spot_policy=spot_policy, + backends=backends, + fleets=fleets, ) endpoint_model = EndpointModel( id=uuid.uuid4(), @@ -157,6 +162,8 @@ async def runner(workspace, request): assert (work_dir / "commands.jsonl").exists() assert (work_dir / "progress.jsonl").exists() assert (work_dir / "hardware_reasoning.md").exists() + assert (work_dir / ".claude" / "skills" / "dstack" / "SKILL.md").exists() + assert (work_dir / ".claude" / "skills" / "dstack-prototyping" / "SKILL.md").exists() final_report = json.loads((work_dir / "final_report.json").read_text()) assert final_report["success"] is True assert final_report["run_name"] == "qwen-agent-candidate" @@ -198,15 +205,11 @@ async def runner(workspace, request): assert "- max_price: 0.3" in prompt assert "- spot_policy: on-demand" in prompt assert "--max-price 0.3 --on-demand" in prompt - assert "printf 'n\\n' | dstack apply" in prompt - assert "Do not use `dstack ps` table text for machine decisions" in prompt - assert "stopped_by_user" in prompt - assert "Do not blindly select the cheapest offer" in prompt assert "--availability-zone" not in prompt assert "--instance " not in prompt @pytest.mark.asyncio - async def test_prompt_includes_real_deployment_harness_context( + async def test_prompt_points_to_bundled_skills_and_endpoint_contract( self, session: AsyncSession, tmp_path, @@ -228,30 +231,70 @@ async def runner(workspace, request): ) ) - endpoint_model = await _create_endpoint_model(session) + endpoint_model = await _create_endpoint_model( + session, + max_price=0.5, + spot_policy=SpotPolicy.ONDEMAND, + backends=[BackendType.RUNPOD], + fleets=["endpoint-e2e-runpod"], + ) service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) assert result.error is None prompt = captured["request"]["prompt"] - assert "This is a real deployment investigation" in prompt - assert "embedded dstack skill guidance" in prompt - assert "embedded dstack-development skill guidance" in prompt + assert "Load and follow `/dstack`" in prompt + assert "Load and follow `/dstack-prototyping`" in prompt + assert "Bundled Claude Code skills are installed in `.claude/skills`" in prompt assert "Do not call hidden server APIs" in prompt - assert "dstack run get --json" in prompt - assert "https://docs.vllm.ai/" in prompt - assert "https://docs.sglang.ai/" in prompt - assert "https://huggingface.co/Qwen/Qwen3-0.6B" in prompt - assert "https://www.wafer.ai/blog/glm52-amd" in prompt - assert "https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development/" in prompt - assert "Prototypes may be dstack services, tasks, or dev environments" in prompt - assert "final verified run reported to the server must be a dstack service" in prompt - assert "do not attempt P/D disaggregation" in prompt + assert "real model API request" in prompt + assert "dstack-development" not in prompt assert "progress.jsonl" in prompt assert ( - "Do not write command output, YAML, secrets, long tables, or raw traces here" in prompt + "Do not write YAML, command output, long tables, raw traces, or secrets to " + "`progress.jsonl`" in prompt + ) + assert ( + "Record each dev environment, task, or service candidate in `candidates.jsonl`" + in prompt + ) + assert "verification.json" in prompt + assert "service run name must not equal the endpoint name `qwen-endpoint`" in prompt + assert "prefer `qwen-endpoint-1`, `qwen-endpoint-2`, etc." in prompt + assert "Make each service YAML self-contained" in prompt + assert "Do not wait only for log text" in prompt + assert "Use normal service logs first" in prompt + assert "- backends: runpod" in prompt + assert ( + "- Reuse these CLI flags where applicable: --max-price 0.5 --on-demand --backend runpod --fleet endpoint-e2e-runpod" + in prompt + ) + assert "RUNPOD" not in prompt + work_dir = tmp_path / str(endpoint_model.id) / "workspace" + prototyping_skill = ( + work_dir / ".claude" / "skills" / "dstack-prototyping" / "SKILL.md" + ).read_text() + assert "Load `/dstack` first" in prototyping_skill + assert "Do not repeat or override `/dstack` command syntax here" in prototyping_skill + assert "Start with vLLM and SGLang" in prototyping_skill + assert "Use a dev environment when an interactive shell can answer" in prototyping_skill + assert "Never collapse tested hardware back into minimum requirements" in prototyping_skill + assert "Do not treat `running`, a passed service probe, or clean logs" in prototyping_skill + assert ( + "Retrying the same YAML after the same error is not prototyping" in prototyping_skill ) + assert "do not name a candidate service exactly like the endpoint" in prototyping_skill + assert "Do not use `:latest` for a final serving image" in prototyping_skill + assert "poll run JSON and stop waiting on terminal states" in prototyping_skill + assert "Use normal logs before diagnostic logs" in prototyping_skill + assert "include applicable backend, fleet, price, spot" in prototyping_skill + assert "https://recipes.vllm.ai/models.json" in prototyping_skill + assert ( + "https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development" + in prototyping_skill + ) + assert "dstack-development" not in prototyping_skill @pytest.mark.asyncio async def test_uses_server_default_agent_budget_when_endpoint_does_not_set_one( @@ -285,6 +328,68 @@ async def runner(workspace, request): assert result.error is None assert captured["request"]["options"]["max_budget"] == 4.0 + @pytest.mark.asyncio + async def test_reuses_existing_final_report_without_invoking_runner( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + endpoint_model = await _create_endpoint_model(session) + work_dir = tmp_path / str(endpoint_model.id) / "workspace" + work_dir.mkdir(parents=True) + run_id = uuid.uuid4() + (work_dir / "final_report.json").write_text( + json.dumps( + { + "success": True, + "run_id": str(run_id), + "run_name": "qwen-agent-candidate", + "service_yaml": "type: service\nname: qwen-agent-candidate\n", + "verification_summary": "Verified before restart.", + } + ), + encoding="utf-8", + ) + + async def runner(workspace, request): + raise AssertionError("runner must not be invoked") + + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + assert result.run_id == run_id + assert result.run_name == "qwen-agent-candidate" + assert result.final_report is not None + assert result.final_report.verification_summary == "Verified before restart." + + @pytest.mark.asyncio + async def test_returns_in_progress_when_agent_process_is_still_running( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(claude_module, "_get_running_agent_process_pid", lambda workspace: 123) + endpoint_model = await _create_endpoint_model(session) + + async def runner(workspace, request): + raise AssertionError("runner must not be invoked") + + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.in_progress is True + assert result.error is None + assert result.final_report is None + @pytest.mark.asyncio async def test_writes_redacted_trace_in_debug_mode( self, @@ -643,15 +748,17 @@ async def flush(self): "json_schema": {}, }, } + (work_dir / "progress.jsonl").write_text( + '{"phase":"old","message":"Old progress must not replay"}\n', + encoding="utf-8", + ) result = await _run_agent_in_subprocess(workspace, request) assert result.error is None assert log_writer.messages == [ - "Starting endpoint provisioning agent (test-model)", "research: Checking recipes with [redacted]", "submit: Submitted service candidate", - "Endpoint provisioning agent finished", ] @pytest.mark.asyncio diff --git a/src/tests/_internal/server/services/test_endpoint_presets.py b/src/tests/_internal/server/services/test_endpoint_presets.py index 57e8980b8b..8f2369f443 100644 --- a/src/tests/_internal/server/services/test_endpoint_presets.py +++ b/src/tests/_internal/server/services/test_endpoint_presets.py @@ -43,11 +43,15 @@ get_job_runtime_data, ) +PROJECT_NAME = "test_project" + class TestLocalDirEndpointPresetService: @pytest.mark.asyncio async def test_lists_valid_endpoint_presets(self, tmp_path): - (tmp_path / "qwen.dstack.yml").write_text( + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen.dstack.yml").write_text( """\ model: Qwen/Qwen3-4B type: endpoint-preset @@ -70,7 +74,7 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): """ ) - presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert len(presets) == 1 assert presets[0].name == "qwen" @@ -83,7 +87,9 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): @pytest.mark.asyncio async def test_lists_replica_group_presets_in_group_order(self, tmp_path): - (tmp_path / "qwen.yml").write_text( + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen.yml").write_text( """\ type: endpoint-preset model: Qwen/Qwen3-4B @@ -131,7 +137,7 @@ async def test_lists_replica_group_presets_in_group_order(self, tmp_path): """ ) - presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert len(presets) == 1 assert [group.name for group in presets[0].replica_spec_groups] == ["router", "worker"] @@ -148,19 +154,21 @@ async def test_saves_preset_without_overwriting(self, tmp_path): service = LocalDirEndpointPresetService(tmp_path) saved = await service.save_preset( + PROJECT_NAME, preset, comments=[ "endpoint: qwen-endpoint", "run: 00000000-0000-0000-0000-000000000001", ], ) - saved_again = await service.save_preset(preset) + saved_again = await service.save_preset(PROJECT_NAME, preset) assert saved.name == "qwen" assert saved_again.name == "qwen-2" - assert (tmp_path / "qwen.dstack.yml").exists() - assert (tmp_path / "qwen-2.dstack.yml").exists() - text = (tmp_path / "qwen.dstack.yml").read_text() + presets_dir = _project_presets_dir(tmp_path) + assert (presets_dir / "qwen.dstack.yml").exists() + assert (presets_dir / "qwen-2.dstack.yml").exists() + text = (presets_dir / "qwen.dstack.yml").read_text() assert text.startswith( "# endpoint: qwen-endpoint\n# run: 00000000-0000-0000-0000-000000000001\n" ) @@ -174,10 +182,11 @@ async def test_saves_preset_without_overwriting(self, tmp_path): @pytest.mark.asyncio async def test_saved_preset_round_trips(self, tmp_path): saved = await LocalDirEndpointPresetService(tmp_path).save_preset( - _qwen_preset(name="Qwen/Qwen3-4B vLLM") + PROJECT_NAME, + _qwen_preset(name="Qwen/Qwen3-4B vLLM"), ) - presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert saved.name == "qwen-qwen3-4b-vllm" assert len(presets) == 1 @@ -192,25 +201,27 @@ async def test_saved_preset_round_trips(self, tmp_path): @pytest.mark.asyncio async def test_deletes_preset_by_name(self, tmp_path): service = LocalDirEndpointPresetService(tmp_path) - saved = await service.save_preset(_qwen_preset(name="qwen")) + saved = await service.save_preset(PROJECT_NAME, _qwen_preset(name="qwen")) - await service.delete_preset(saved.name) + await service.delete_preset(PROJECT_NAME, saved.name) - assert await service.list_presets() == [] - assert not (tmp_path / "qwen.dstack.yml").exists() + assert await service.list_presets(PROJECT_NAME) == [] + assert not (_project_presets_dir(tmp_path) / "qwen.dstack.yml").exists() @pytest.mark.asyncio async def test_delete_missing_preset_raises(self, tmp_path): service = LocalDirEndpointPresetService(tmp_path) with pytest.raises(FileNotFoundError): - await service.delete_preset("missing") + await service.delete_preset(PROJECT_NAME, "missing") @pytest.mark.asyncio async def test_ignores_non_yaml_files(self, tmp_path): - (tmp_path / "notes.txt").write_text("ignored") + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "notes.txt").write_text("ignored") - presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert presets == [] @@ -416,9 +427,11 @@ async def test_ignores_non_yaml_files(self, tmp_path): async def test_invalid_preset_is_skipped_and_logged( self, tmp_path, caplog, filename, content, message ): - (tmp_path / filename).write_text(content) + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / filename).write_text(content) - presets = await LocalDirEndpointPresetService(tmp_path).list_presets() + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert presets == [] assert filename in caplog.text @@ -468,7 +481,7 @@ async def test_builds_implicit_replica_group_from_running_service( await session.refresh(run, attribute_names=["jobs"]) preset = build_endpoint_preset_from_run("qwen-learned", run) - saved = await LocalDirEndpointPresetService(tmp_path).save_preset(preset) + saved = await LocalDirEndpointPresetService(tmp_path).save_preset(PROJECT_NAME, preset) assert saved.name == "qwen-learned" assert saved.model == "Qwen/Qwen3-4B" @@ -591,7 +604,7 @@ async def test_builds_replica_groups_in_service_order(self, session: AsyncSessio await session.refresh(run, attribute_names=["jobs"]) preset = build_endpoint_preset_from_run("qwen-grouped", run) - saved = await LocalDirEndpointPresetService(tmp_path).save_preset(preset) + saved = await LocalDirEndpointPresetService(tmp_path).save_preset(PROJECT_NAME, preset) assert [group.name for group in saved.replica_spec_groups] == ["router", "worker"] assert "cpu=4" in saved.replica_spec_groups[0].resources.pretty_format() @@ -694,7 +707,9 @@ def test_builds_repo_less_run_spec(self): @pytest.mark.asyncio async def test_missing_dir_returns_no_presets(self, tmp_path): - presets = await LocalDirEndpointPresetService(tmp_path / "missing").list_presets() + presets = await LocalDirEndpointPresetService(tmp_path / "missing").list_presets( + PROJECT_NAME + ) assert presets == [] @@ -799,7 +814,9 @@ async def test_matching_plan_requires_available_offers(self, session: AsyncSessi async def test_skips_preset_with_unresolved_env_sentinel( self, session: AsyncSession, tmp_path ): - (tmp_path / "qwen.yml").write_text( + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen.yml").write_text( """\ type: endpoint-preset model: Qwen/Qwen3-4B @@ -884,7 +901,9 @@ async def test_refreshes_user_ssh_key_before_planning(self, session: AsyncSessio def _write_qwen_preset(tmp_path): - (tmp_path / "qwen.yml").write_text( + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen.yml").write_text( """\ type: endpoint-preset model: Qwen/Qwen3-4B @@ -907,6 +926,10 @@ def _write_qwen_preset(tmp_path): ) +def _project_presets_dir(projects_dir): + return projects_dir / PROJECT_NAME / "presets" + + def _instance_offer( gpu_name: str = "T4", gpu_memory_gib: float = 16, From 6a971e97a7554d1379c64ebd9cd61e6dd0d06748 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 6 Jul 2026 13:55:52 +0200 Subject: [PATCH 14/21] Hide stopped endpoint backing run --- endpoint-e2e-testing-report.md | 5 ++ .../server/services/endpoints/__init__.py | 6 ++- .../server/routers/test_endpoints.py | 46 ++++++++++++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/endpoint-e2e-testing-report.md b/endpoint-e2e-testing-report.md index 5914a94c31..af91e47650 100644 --- a/endpoint-e2e-testing-report.md +++ b/endpoint-e2e-testing-report.md @@ -378,6 +378,11 @@ Stop result: Result: `dstack endpoint stop` stopped the backing service run, terminated the RunPod instance, and left the endpoint visible as `stopped`. +Follow-up UX cleanup: stopped endpoints now hide the linked service run in `dstack endpoint` and +`get --json` (`RUN` displays `-`, `run_name` is `null`). The internal `service_run_id` remains in +the database for lifecycle/history purposes, but the user-facing field represents the current live +backing service, not old run history. + ## Assessment This was the first meaningful proof that the endpoint idea can work end to end: endpoint config in, real agent work, real service deployed, real model request verified, endpoint running, preset saved. diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py index 7a571a9fb9..fcba3ce103 100644 --- a/src/dstack/_internal/server/services/endpoints/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -494,7 +494,11 @@ def endpoint_model_to_endpoint(endpoint_model: EndpointModel) -> Endpoint: run_name = None url = None status = endpoint_model.status - if endpoint_model.service_run is not None and not endpoint_model.service_run.deleted: + if ( + status != EndpointStatus.STOPPED + and endpoint_model.service_run is not None + and not endpoint_model.service_run.deleted + ): run_name = endpoint_model.service_run.run_name if ( status == EndpointStatus.RUNNING diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index d7e209f0ba..b6058541b7 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -15,7 +15,7 @@ InstanceAvailability, InstanceOfferWithAvailability, ) -from dstack._internal.core.models.runs import RunSpec +from dstack._internal.core.models.runs import RunSpec, RunStatus from dstack._internal.core.models.users import GlobalRole, ProjectRole from dstack._internal.server import settings from dstack._internal.server.models import EndpointModel @@ -31,6 +31,8 @@ from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( create_project, + create_repo, + create_run, create_user, get_auth_headers, list_events, @@ -380,6 +382,48 @@ async def test_returns_40x_if_not_authenticated(self, client: AsyncClient): response = await client.post("/api/project/main/endpoints/get") assert response.status_code in [401, 403] + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_stopped_endpoint_hides_linked_run_name( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + endpoint_model = await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.STOPPED, + ) + repo = await create_repo( + session=session, + project_id=project.id, + repo_name="qwen-endpoint-serving-repo", + ) + run = await create_run( + session=session, + project=project, + repo=repo, + user=user, + run_name="qwen-endpoint-serving", + status=RunStatus.TERMINATED, + ) + endpoint_model.service_run_id = run.id + await session.commit() + + response = await client.post( + f"/api/project/{project.name}/endpoints/get", + headers=get_auth_headers(user.token), + json={"name": endpoint_model.name}, + ) + + assert response.status_code == 200, response.json() + assert response.json()["status"] == "stopped" + assert response.json()["run_name"] is None + class TestStopEndpoint: @pytest.mark.asyncio From 18db03b7652adda2ec0765dd7e2e8d484b3a94c2 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 6 Jul 2026 15:12:00 +0200 Subject: [PATCH 15/21] Add persistent endpoint agent attempts --- .../background/pipeline_tasks/endpoints.py | 177 ++++- ...e1d6c2a9b77_add_endpoint_agent_attempts.py | 77 ++ src/dstack/_internal/server/models.py | 41 ++ .../server/services/endpoints/__init__.py | 85 ++- .../services/endpoints/agent/__init__.py | 1 + .../server/services/endpoints/agent/claude.py | 674 ++++++++++++++++-- .../pipeline_tasks/test_endpoints.py | 260 ++++++- .../server/routers/test_endpoints.py | 117 ++- .../services/endpoints/test_claude_agent.py | 172 ++++- 9 files changed, 1506 insertions(+), 98 deletions(-) create mode 100644 src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py index 923a824fcf..64082c1db1 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -32,10 +32,18 @@ set_unlock_update_map_fields, ) from dstack._internal.server.db import get_db, get_session_ctx -from dstack._internal.server.models import EndpointModel, ProjectModel, RunModel, UserModel +from dstack._internal.server.models import ( + EndpointModel, + EndpointRunSubmissionModel, + ProjectModel, + RunModel, + UserModel, +) from dstack._internal.server.services import runs as runs_services from dstack._internal.server.services.endpoints import ( + can_use_endpoint_agent, emit_endpoint_status_change_event, + get_endpoint_agent_admin_required_message, get_endpoint_configuration, record_endpoint_run_submission, ) @@ -256,8 +264,8 @@ async def process(self, item: EndpointPipelineItem): else: result = _ProcessResult() - run_to_stop = await _get_backing_run_to_stop_after_failure(endpoint_model, result) - if run_to_stop is not None: + runs_to_stop = await _get_endpoint_runs_to_stop_after_failure(endpoint_model, result) + for run_to_stop in runs_to_stop: logger.info( "Stopping backing run %s after endpoint %s failed", run_to_stop.run_name, @@ -281,7 +289,14 @@ async def _refetch_locked_endpoint(item: EndpointPipelineItem) -> Optional[Endpo EndpointModel.lock_token == item.lock_token, ) .options(joinedload(EndpointModel.project).joinedload(ProjectModel.backends)) - .options(joinedload(EndpointModel.user).load_only(UserModel.name, UserModel.token)) + .options(joinedload(EndpointModel.project).joinedload(ProjectModel.members)) + .options( + joinedload(EndpointModel.user).load_only( + UserModel.name, + UserModel.global_role, + UserModel.token, + ) + ) .options(joinedload(EndpointModel.service_run).selectinload(RunModel.jobs)) ) return res.unique().scalar_one_or_none() @@ -304,13 +319,19 @@ async def _apply_process_result( .where( EndpointModel.id == endpoint_model.id, EndpointModel.lock_token == endpoint_model.lock_token, + EndpointModel.status == item.status, ) .values(**update_map) .returning(EndpointModel.id) ) updated_ids = list(res.scalars().all()) if len(updated_ids) == 0: - log_lock_token_changed_after_processing(logger, item) + if await _link_reported_service_run_to_stopping_endpoint(session, item, result): + return + logger.info( + "Endpoint %s changed while being processed; ignoring stale result", + endpoint_model.name, + ) return emit_endpoint_status_change_event( session=session, @@ -321,6 +342,43 @@ async def _apply_process_result( ) +async def _link_reported_service_run_to_stopping_endpoint( + session, + item: EndpointPipelineItem, + result: "_ProcessResult", +) -> bool: + service_run_id = result.update_map.get("service_run_id") + if service_run_id is None: + return False + + update_map = _EndpointUpdateMap(service_run_id=service_run_id) + set_processed_update_map_fields(update_map) + set_unlock_update_map_fields(update_map) + + resolve_now_placeholders(update_map, now=get_current_datetime()) + res = await session.execute( + update(EndpointModel) + .where( + EndpointModel.id == item.id, + EndpointModel.lock_token == item.lock_token, + EndpointModel.status == EndpointStatus.STOPPING, + EndpointModel.service_run_id.is_(None), + ) + .values(**update_map) + .returning(EndpointModel.id) + ) + updated_ids = list(res.scalars().all()) + if len(updated_ids) == 0: + log_lock_token_changed_after_processing(logger, item) + return False + logger.info( + "Linked reported service run %s to stopping endpoint %s", + service_run_id, + item.id, + ) + return True + + class _EndpointUpdateMap(ItemUpdateMap, total=False): status: EndpointStatus status_message: Optional[str] @@ -345,18 +403,39 @@ class _PresetSubmissionResult: unprovisionable_preset_name: Optional[str] = None -async def _get_backing_run_to_stop_after_failure( +async def _get_endpoint_runs_to_stop_after_failure( endpoint_model: EndpointModel, result: _ProcessResult, -) -> Optional[RunModel]: +) -> list[RunModel]: if result.update_map.get("status") != EndpointStatus.FAILED: - return None - run_model = endpoint_model.service_run - if run_model is None or run_model.deleted: - return None - if run_model.status.is_finished() or run_model.status == RunStatus.TERMINATING: - return None - return run_model + return [] + return [ + run + for run in await _get_endpoint_unfinished_runs(endpoint_model) + if run.status != RunStatus.TERMINATING + ] + + +async def _get_endpoint_unfinished_runs(endpoint_model: EndpointModel) -> list[RunModel]: + runs: list[RunModel] = [] + seen_run_ids: set[uuid.UUID] = set() + if endpoint_model.service_run is not None: + runs.append(endpoint_model.service_run) + seen_run_ids.add(endpoint_model.service_run.id) + async with get_session_ctx() as session: + res = await session.execute( + select(RunModel) + .join( + EndpointRunSubmissionModel, + EndpointRunSubmissionModel.run_id == RunModel.id, + ) + .where( + EndpointRunSubmissionModel.endpoint_id == endpoint_model.id, + RunModel.deleted == False, + ) + ) + runs.extend(run for run in res.unique().scalars().all() if run.id not in seen_run_ids) + return [run for run in runs if not run.deleted and not run.status.is_finished()] async def _process_submitted_endpoint( @@ -415,6 +494,7 @@ def _should_provision_with_agent(endpoint_model: EndpointModel) -> bool: return ( endpoint_configuration.preset_policy != EndpointPresetPolicy.REUSE and get_agent_service().is_enabled() + and can_use_endpoint_agent(user=endpoint_model.user, project=endpoint_model.project) ) @@ -538,6 +618,10 @@ async def _provision_endpoint_with_agent( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) + await _record_agent_candidate_run_submissions( + endpoint_model=endpoint_model, + candidate_run_ids=result.candidate_run_ids, + ) if result.in_progress: return _ProcessResult( update_map={ @@ -705,6 +789,48 @@ async def _record_endpoint_run_submission(endpoint_id: uuid.UUID, run_id: uuid.U await session.commit() +async def _record_agent_candidate_run_submissions( + *, + endpoint_model: EndpointModel, + candidate_run_ids: Sequence[uuid.UUID], +) -> None: + if len(candidate_run_ids) == 0: + return + async with get_session_ctx() as session: + res = await session.execute( + select(RunModel).where( + RunModel.id.in_(candidate_run_ids), + RunModel.project_id == endpoint_model.project_id, + RunModel.user_id == endpoint_model.user_id, + RunModel.deleted == False, + ) + ) + runs_by_id = {run.id: run for run in res.scalars().all()} + for run_id in candidate_run_ids: + if run_id not in runs_by_id: + logger.info( + "Ignoring endpoint %s candidate run %s because it is not a live run " + "owned by the endpoint user/project", + endpoint_model.name, + run_id, + ) + continue + try: + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run_id, + ) + except ServerClientError as e: + logger.info( + "Ignoring endpoint %s candidate run %s: %s", + endpoint_model.name, + run_id, + e.msg, + ) + await session.commit() + + async def _process_running_endpoint(endpoint_model: EndpointModel) -> _ProcessResult: readiness = _get_backing_service_readiness(endpoint_model) if readiness.failed_message is not None: @@ -723,12 +849,14 @@ async def _process_stopping_endpoint( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: - run_model = endpoint_model.service_run # TODO: When the Claude agent service exposes cancellation, interrupt a live # agent process here instead of waiting for it to notice/finish. - if run_model is None or run_model.deleted or run_model.status.is_finished(): + run_models = await _get_endpoint_unfinished_runs(endpoint_model) + if not run_models: return _get_stopped_result() - if run_model.status != RunStatus.TERMINATING: + for run_model in run_models: + if run_model.status == RunStatus.TERMINATING: + continue logger.info( "Stopping backing run %s before stopping endpoint %s", run_model.run_name, @@ -930,6 +1058,11 @@ def _get_no_provisioning_path_message( ) if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE: return reason + if get_agent_service().is_enabled() and not can_use_endpoint_agent( + user=endpoint_model.user, + project=endpoint_model.project, + ): + return f"{reason} {get_endpoint_agent_admin_required_message()}" agent_unavailable_reason = get_agent_unavailable_reason() if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: return ( @@ -938,6 +1071,16 @@ def _get_no_provisioning_path_message( ) if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE: return _NO_MATCHING_PRESET_MESSAGE + if get_agent_service().is_enabled() and not can_use_endpoint_agent( + user=endpoint_model.user, + project=endpoint_model.project, + ): + if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: + return ( + "No matching endpoint presets found. " + f"{get_endpoint_agent_admin_required_message()}" + ) + return get_endpoint_agent_admin_required_message() agent_unavailable_reason = get_agent_unavailable_reason() if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: return ( diff --git a/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py b/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py new file mode 100644 index 0000000000..907d4058e3 --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py @@ -0,0 +1,77 @@ +"""add endpoint agent attempts + +Revision ID: 4e1d6c2a9b77 +Revises: 03efb71a1563 +Create Date: 2026-07-06 17:00:00.000000+00:00 + +""" + +import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op + +import dstack._internal.server.models + +# revision identifiers, used by Alembic. +revision = "4e1d6c2a9b77" +down_revision = "03efb71a1563" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "endpoint_agent_attempts", + sa.Column( + "endpoint_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False + ), + sa.Column("attempt_num", sa.Integer(), nullable=False), + sa.Column("status", sa.String(length=100), nullable=False), + sa.Column("workspace_path", sa.Text(), nullable=False), + sa.Column("pid", sa.Integer(), nullable=True), + sa.Column("process_host", sa.String(length=255), nullable=True), + sa.Column("progress_log_offset", sa.BigInteger(), nullable=False), + sa.Column("stdout_log_offset", sa.BigInteger(), nullable=False), + sa.Column("stderr_log_offset", sa.BigInteger(), nullable=False), + sa.Column("max_agent_budget", sa.Float(), nullable=True), + sa.Column("spent_agent_budget", sa.Float(), nullable=True), + sa.Column("status_message", sa.Text(), nullable=True), + sa.Column("created_at", dstack._internal.server.models.NaiveDateTime(), nullable=False), + sa.Column("updated_at", dstack._internal.server.models.NaiveDateTime(), nullable=False), + sa.Column("finished_at", dstack._internal.server.models.NaiveDateTime(), nullable=True), + sa.ForeignKeyConstraint( + ["endpoint_id"], + ["endpoints.id"], + name=op.f("fk_endpoint_agent_attempts_endpoint_id_endpoints"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "endpoint_id", "attempt_num", name=op.f("pk_endpoint_agent_attempts") + ), + ) + with op.batch_alter_table("endpoint_agent_attempts", schema=None) as batch_op: + batch_op.create_index( + batch_op.f("ix_endpoint_agent_attempts_endpoint_id"), ["endpoint_id"], unique=False + ) + batch_op.create_index( + "ix_endpoint_agent_attempts_endpoint_status", + ["endpoint_id", "status"], + unique=False, + ) + batch_op.create_index( + batch_op.f("ix_endpoint_agent_attempts_status"), ["status"], unique=False + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("endpoint_agent_attempts", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_endpoint_agent_attempts_status")) + batch_op.drop_index("ix_endpoint_agent_attempts_endpoint_status") + batch_op.drop_index(batch_op.f("ix_endpoint_agent_attempts_endpoint_id")) + + op.drop_table("endpoint_agent_attempts") + # ### end Alembic commands ### diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 29595a0c46..ec9770411f 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -55,6 +55,12 @@ CASCADE_DEFAULT_WITH_DELETE_ORPHAN = "save-update, merge, delete-orphan, delete" +class EndpointAgentAttemptStatus(enum.Enum): + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + class NaiveDateTime(TypeDecorator): """ A custom type decorator that ensures datetime objects are offset-naive when stored in the database @@ -1061,6 +1067,41 @@ class EndpointRunSubmissionModel(BaseModel): ) +class EndpointAgentAttemptModel(BaseModel): + __tablename__ = "endpoint_agent_attempts" + + endpoint_id: Mapped[uuid.UUID] = mapped_column( + ForeignKey("endpoints.id", ondelete="CASCADE"), primary_key=True + ) + endpoint: Mapped["EndpointModel"] = relationship() + + attempt_num: Mapped[int] = mapped_column(Integer, primary_key=True) + status: Mapped[EndpointAgentAttemptStatus] = mapped_column( + EnumAsString(EndpointAgentAttemptStatus, 100), index=True + ) + workspace_path: Mapped[str] = mapped_column(Text) + + pid: Mapped[Optional[int]] = mapped_column(Integer) + process_host: Mapped[Optional[str]] = mapped_column(String(255)) + + progress_log_offset: Mapped[int] = mapped_column(BigInteger, default=0) + stdout_log_offset: Mapped[int] = mapped_column(BigInteger, default=0) + stderr_log_offset: Mapped[int] = mapped_column(BigInteger, default=0) + + max_agent_budget: Mapped[Optional[float]] = mapped_column(Float) + spent_agent_budget: Mapped[Optional[float]] = mapped_column(Float) + status_message: Mapped[Optional[str]] = mapped_column(Text) + + created_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime) + updated_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime) + finished_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) + + __table_args__ = ( + Index("ix_endpoint_agent_attempts_endpoint_id", endpoint_id), + Index("ix_endpoint_agent_attempts_endpoint_status", endpoint_id, status), + ) + + class VolumeAttachmentModel(BaseModel): __tablename__ = "volumes_attachments" diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py index fcba3ce103..2f8a9c3faf 100644 --- a/src/dstack/_internal/server/services/endpoints/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload -from dstack._internal.core.errors import ResourceExistsError, ServerClientError +from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError from dstack._internal.core.models.common import ApplyAction from dstack._internal.core.models.endpoints import ( Endpoint, @@ -21,6 +21,7 @@ EndpointStatus, ) from dstack._internal.core.models.runs import ServiceSpec +from dstack._internal.core.models.users import GlobalRole, ProjectRole from dstack._internal.core.services import validate_dstack_resource_name from dstack._internal.server.db import get_db, is_db_postgres, is_db_sqlite from dstack._internal.server.models import ( @@ -42,12 +43,19 @@ ) from dstack._internal.server.services.locking import get_locker, string_to_lock_id from dstack._internal.server.services.pipelines import PipelineHinterProtocol -from dstack._internal.server.services.projects import list_user_project_models +from dstack._internal.server.services.projects import ( + get_user_project_role, + list_user_project_models, +) from dstack._internal.utils import common, random_names from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) +_ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE = ( + "Creating endpoint presets with the server agent requires project admin permissions." +) + def switch_endpoint_status( session: AsyncSession, @@ -289,11 +297,17 @@ async def get_endpoint_plan( configuration.preset_policy != EndpointPresetPolicy.REUSE and get_agent_service().is_enabled() ): - provisioning_plan = _agent_plan_to_provisioning_plan( - get_agent_service().get_plan(), - max_budget=get_effective_max_agent_budget(configuration), - reason=_get_unprovisionable_preset_reason(unprovisionable_preset_plan), - ) + if can_use_endpoint_agent(user=user, project=project): + provisioning_plan = _agent_plan_to_provisioning_plan( + get_agent_service().get_plan(), + max_budget=get_effective_max_agent_budget(configuration), + reason=_get_unprovisionable_preset_reason(unprovisionable_preset_plan), + ) + else: + provisioning_plan = _get_agent_forbidden_provisioning_plan( + configuration.preset_policy, + unprovisionable_preset_plan=unprovisionable_preset_plan, + ) return EndpointPlan( project_name=project.name, user=user.name, @@ -306,6 +320,16 @@ async def get_endpoint_plan( ) +def can_use_endpoint_agent(user: UserModel, project: ProjectModel) -> bool: + if user.global_role == GlobalRole.ADMIN: + return True + return get_user_project_role(user=user, project=project) == ProjectRole.ADMIN + + +def get_endpoint_agent_admin_required_message() -> str: + return _ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE + + def _agent_plan_to_provisioning_plan( agent_plan: AgentPlan, max_budget: Optional[float], @@ -348,6 +372,24 @@ def _get_no_provisioning_plan( ) +def _get_agent_forbidden_provisioning_plan( + preset_policy: EndpointPresetPolicy, + unprovisionable_preset_plan: Optional[EndpointPresetPlan] = None, +) -> EndpointProvisioningPlanNone: + preset_reason = _get_unprovisionable_preset_reason(unprovisionable_preset_plan) + if preset_reason is not None: + return EndpointProvisioningPlanNone( + reason=f"{preset_reason} {_ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE}" + ) + if preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: + return EndpointProvisioningPlanNone( + reason=( + f"No matching endpoint presets found. {_ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE}" + ) + ) + return EndpointProvisioningPlanNone(reason=_ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE) + + def _get_unprovisionable_preset_reason( preset_plan: Optional[EndpointPresetPlan], ) -> Optional[str]: @@ -395,6 +437,12 @@ async def create_endpoint( pipeline_hinter: PipelineHinterProtocol, ) -> Endpoint: _validate_endpoint_configuration(configuration) + await _check_can_submit_endpoint_configuration( + session=session, + project=project, + user=user, + configuration=configuration, + ) lock_namespace = f"endpoint_names_{project.name}" if is_db_sqlite(): @@ -461,6 +509,29 @@ async def create_endpoint( return endpoint_model_to_endpoint(endpoint_model) +async def _check_can_submit_endpoint_configuration( + session: AsyncSession, + project: ProjectModel, + user: UserModel, + configuration: EndpointConfiguration, +) -> None: + if configuration.preset_policy == EndpointPresetPolicy.REUSE: + return + if can_use_endpoint_agent(user=user, project=project): + return + if configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: + preset_planning_result = await find_preset_planning_result( + session=session, + project=project, + user=user, + endpoint_name=configuration.name, + endpoint_configuration=configuration, + ) + if preset_planning_result.provisionable is not None: + return + raise ForbiddenError(_ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE) + + async def stop_endpoints( session: AsyncSession, project: ProjectModel, diff --git a/src/dstack/_internal/server/services/endpoints/agent/__init__.py b/src/dstack/_internal/server/services/endpoints/agent/__init__.py index 3bef1d5043..7d70a941ba 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/agent/__init__.py @@ -19,6 +19,7 @@ class AgentPlan: class AgentProvisioningResult: run_id: Optional[uuid.UUID] = None run_name: Optional[str] = None + candidate_run_ids: tuple[uuid.UUID, ...] = () error: Optional[str] = None final_report: Optional[AgentFinalReport] = None in_progress: bool = False diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index 903064b4a8..56523483d8 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -3,6 +3,8 @@ import json import os import shutil +import socket +import subprocess from contextlib import suppress from dataclasses import dataclass from datetime import datetime, timezone @@ -12,11 +14,18 @@ import yaml from pydantic import ValidationError +from sqlalchemy import func, select from dstack._internal.core.models.config import GlobalConfig, ProjectConfig from dstack._internal.core.models.endpoints import EndpointConfiguration from dstack._internal.server import settings -from dstack._internal.server.models import EndpointModel, ProjectModel +from dstack._internal.server.db import get_session_ctx +from dstack._internal.server.models import ( + EndpointAgentAttemptModel, + EndpointAgentAttemptStatus, + EndpointModel, + ProjectModel, +) from dstack._internal.server.schemas.runner import LogEvent from dstack._internal.server.services import logs as logs_services from dstack._internal.server.services.endpoints.agent import ( @@ -30,7 +39,11 @@ AgentFinalReport, ) from dstack._internal.server.services.pipelines import PipelineHinterProtocol -from dstack._internal.utils.common import get_milliseconds_since_epoch, run_async +from dstack._internal.utils.common import ( + get_current_datetime, + get_milliseconds_since_epoch, + run_async, +) from dstack._internal.utils.logging import get_logger logger = get_logger(__name__) @@ -55,6 +68,8 @@ _AGENT_LOG_BATCH_SIZE = 1 _AGENT_PROGRESS_LOG_NAME = "progress.jsonl" _AGENT_PROCESS_STATE_NAME = "agent_process.json" +_AGENT_STDOUT_LOG_NAME = "agent_stdout.jsonl" +_AGENT_STDERR_LOG_NAME = "agent_stderr.jsonl" _AGENT_PROGRESS_POLL_SECONDS = 1.0 _CLAUDE_AGENT_TOOLS = "Bash,Read,Write,Edit,WebFetch,WebSearch,StructuredOutput" @@ -69,6 +84,7 @@ class _AgentRunnerResult: class _AgentProcessOutput: report_data: Optional[dict[str, Any]] = None result_error: Optional[str] = None + spent_budget: Optional[float] = None stdout_tail: str = "" @@ -80,7 +96,7 @@ def __init__( ] = None, workspace_base_dir: Path = settings.SERVER_DATA_DIR_PATH / "endpoint_agent_runs", ) -> None: - self._runner = runner or _run_agent_in_subprocess + self._runner = runner self._workspace_base_dir = workspace_base_dir def is_enabled(self) -> bool: @@ -99,9 +115,17 @@ async def provision_endpoint( return AgentProvisioningResult(error=unavailable_reason) try: - workspace = _prepare_workspace( + max_agent_budget = get_effective_max_agent_budget( + EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) + ) + attempt = await _get_or_create_agent_attempt( endpoint_model=endpoint_model, workspace_base_dir=self._workspace_base_dir, + max_agent_budget=max_agent_budget, + ) + workspace = _prepare_workspace( + endpoint_model=endpoint_model, + workspace_root_dir=Path(attempt.workspace_path), ) except Exception as e: logger.warning("Failed to prepare endpoint agent workspace: %s", e, exc_info=True) @@ -109,49 +133,440 @@ async def provision_endpoint( error=f"Failed to prepare endpoint agent workspace: {e}" ) - existing_report = _load_final_report(workspace) - if existing_report is not None: - workspace.artifacts.record_report(existing_report) - return AgentProvisioningResult( - run_id=existing_report.run_id, - run_name=existing_report.run_name, - final_report=existing_report, + if self._runner is not None: + return await _run_injected_agent_runner( + attempt=attempt, + workspace=workspace, + runner=self._runner, ) + return await _reconcile_detached_agent_attempt( + attempt=attempt, + workspace=workspace, + ) - running_pid = _get_running_agent_process_pid(workspace) - if running_pid is not None: - logger.info( - "Endpoint agent process %s is already running for endpoint %s", - running_pid, - endpoint_model.name, - ) - return AgentProvisioningResult(in_progress=True) - - runner_result = await self._runner(workspace, _build_agent_request(workspace)) - if runner_result.error is not None: - workspace.artifacts.record_error(runner_result.error) - return AgentProvisioningResult(error=runner_result.error) - report = runner_result.report - if report is None: - workspace.artifacts.record_error("Server agent did not return a verification report") + +async def _run_injected_agent_runner( + *, + attempt: EndpointAgentAttemptModel, + workspace: "_AgentWorkspace", + runner: Callable[["_AgentWorkspace", dict[str, Any]], Awaitable[_AgentRunnerResult]], +) -> AgentProvisioningResult: + reconcile_result = await _reconcile_agent_artifacts(attempt=attempt, workspace=workspace) + if reconcile_result is not None: + return reconcile_result + + runner_result = await runner(workspace, _build_agent_request(workspace)) + candidate_run_ids = _load_candidate_run_ids(workspace) + if runner_result.error is not None: + workspace.artifacts.record_error(runner_result.error) + await _mark_agent_attempt_failed(attempt, runner_result.error) + return AgentProvisioningResult( + error=runner_result.error, + candidate_run_ids=candidate_run_ids, + ) + report = runner_result.report + if report is None: + error = "Server agent did not return a verification report" + workspace.artifacts.record_error(error) + await _mark_agent_attempt_failed(attempt, error) + return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) + workspace.artifacts.record_report(report) + await _mark_agent_attempt_from_report(attempt, report, spent_budget=None) + + logger.info( + "Endpoint agent finished for endpoint %s: success=%s run_id=%s run_name=%s", + workspace.endpoint_name, + report.success, + report.run_id, + report.run_name, + ) + return AgentProvisioningResult( + run_id=report.run_id, + run_name=report.run_name, + candidate_run_ids=candidate_run_ids, + final_report=report, + ) + + +async def _reconcile_detached_agent_attempt( + *, + attempt: EndpointAgentAttemptModel, + workspace: "_AgentWorkspace", +) -> AgentProvisioningResult: + reconcile_result = await _reconcile_agent_artifacts(attempt=attempt, workspace=workspace) + if reconcile_result is not None: + return reconcile_result + + running_pid = _get_running_agent_process_pid(workspace, attempt=attempt) + if running_pid is not None: + logger.info( + "Endpoint agent process %s is already running for endpoint %s", + running_pid, + workspace.endpoint_name, + ) + return AgentProvisioningResult( + in_progress=True, + candidate_run_ids=_load_candidate_run_ids(workspace), + ) + + if attempt.pid is not None: + error = "Server agent process exited without a verification report" + workspace.artifacts.record_error(error) + await _mark_agent_attempt_failed(attempt, error) + return AgentProvisioningResult( + error=error, + candidate_run_ids=_load_candidate_run_ids(workspace), + ) + + try: + pid = _start_agent_subprocess_detached(workspace, _build_agent_request(workspace)) + except Exception as e: + logger.warning("Failed to start endpoint agent process: %s", e, exc_info=True) + error = f"Failed to start endpoint agent process: {e}" + workspace.artifacts.record_error(error) + await _mark_agent_attempt_failed(attempt, error) + return AgentProvisioningResult(error=error) + + await _mark_agent_attempt_running(attempt, pid=pid, process_host=_get_process_host()) + return AgentProvisioningResult(in_progress=True) + + +async def _reconcile_agent_artifacts( + *, + attempt: EndpointAgentAttemptModel, + workspace: "_AgentWorkspace", +) -> Optional[AgentProvisioningResult]: + process_output = await _reconcile_agent_stream_files(attempt=attempt, workspace=workspace) + await _reconcile_agent_progress(attempt=attempt, workspace=workspace) + candidate_run_ids = _load_candidate_run_ids(workspace) + + report = _load_final_report(workspace) + if report is None and process_output.report_data is not None: + try: + report = AgentFinalReport.parse_obj(process_output.report_data) + except ValidationError as e: + error = f"Server agent returned an invalid verification report: {e}" + workspace.artifacts.record_error(error) + await _mark_agent_attempt_failed(attempt, error) + return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) + if report is not None: + if ( + process_output.spent_budget is None + and _get_running_agent_process_pid(workspace, attempt=attempt) is not None + ): return AgentProvisioningResult( - error="Server agent did not return a verification report" + in_progress=True, + candidate_run_ids=candidate_run_ids, ) workspace.artifacts.record_report(report) - - logger.info( - "Endpoint agent finished for endpoint %s: success=%s run_id=%s run_name=%s", - endpoint_model.name, - report.success, - report.run_id, - report.run_name, + await _mark_agent_attempt_from_report( + attempt, + report, + spent_budget=process_output.spent_budget, ) return AgentProvisioningResult( run_id=report.run_id, run_name=report.run_name, + candidate_run_ids=candidate_run_ids, final_report=report, ) + error = _load_agent_error(workspace) + if error is not None: + await _mark_agent_attempt_failed( + attempt, + error, + spent_budget=process_output.spent_budget, + ) + return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) + if process_output.result_error is not None: + error = "Server agent failed before returning a verification report" + workspace.artifacts.record_error(error) + await _mark_agent_attempt_failed( + attempt, + error, + spent_budget=process_output.spent_budget, + ) + return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) + return None + + +async def _get_or_create_agent_attempt( + *, + endpoint_model: EndpointModel, + workspace_base_dir: Path, + max_agent_budget: Optional[float], +) -> EndpointAgentAttemptModel: + async with get_session_ctx() as session: + res = await session.execute( + select(EndpointAgentAttemptModel) + .where(EndpointAgentAttemptModel.endpoint_id == endpoint_model.id) + .order_by(EndpointAgentAttemptModel.attempt_num.desc()) + .limit(1) + ) + attempt = res.scalar_one_or_none() + if attempt is not None and attempt.created_at >= endpoint_model.created_at: + return attempt + + max_attempt_num_res = await session.execute( + select(func.max(EndpointAgentAttemptModel.attempt_num)).where( + EndpointAgentAttemptModel.endpoint_id == endpoint_model.id + ) + ) + attempt_num = (max_attempt_num_res.scalar_one_or_none() or 0) + 1 + now = get_current_datetime() + legacy_workspace_path = workspace_base_dir / str(endpoint_model.id) + if attempt_num == 1 and _has_agent_workspace_artifacts(legacy_workspace_path): + workspace_path = legacy_workspace_path + else: + workspace_path = workspace_base_dir / str(endpoint_model.id) / str(attempt_num) + attempt = EndpointAgentAttemptModel( + endpoint_id=endpoint_model.id, + attempt_num=attempt_num, + status=EndpointAgentAttemptStatus.RUNNING, + workspace_path=str(workspace_path), + progress_log_offset=0, + stdout_log_offset=0, + stderr_log_offset=0, + max_agent_budget=max_agent_budget, + created_at=now, + updated_at=now, + ) + session.add(attempt) + await session.commit() + return attempt + + +def _has_agent_workspace_artifacts(root_dir: Path) -> bool: + work_dir = root_dir / "workspace" + return any( + (work_dir / filename).exists() + for filename in [ + "final_report.json", + "agent_error.json", + _AGENT_PROCESS_STATE_NAME, + _AGENT_PROGRESS_LOG_NAME, + ] + ) + + +async def _get_agent_attempt(session, attempt: EndpointAgentAttemptModel): + return await session.get( + EndpointAgentAttemptModel, + { + "endpoint_id": attempt.endpoint_id, + "attempt_num": attempt.attempt_num, + }, + ) + + +async def _mark_agent_attempt_running( + attempt: EndpointAgentAttemptModel, + *, + pid: int, + process_host: str, +) -> None: + async with get_session_ctx() as session: + stored_attempt = await _get_agent_attempt(session, attempt) + if stored_attempt is None: + return + stored_attempt.status = EndpointAgentAttemptStatus.RUNNING + stored_attempt.pid = pid + stored_attempt.process_host = process_host + stored_attempt.updated_at = get_current_datetime() + await session.commit() + attempt.pid = pid + attempt.process_host = process_host + + +async def _mark_agent_attempt_from_report( + attempt: EndpointAgentAttemptModel, + report: AgentFinalReport, + *, + spent_budget: Optional[float], +) -> None: + if report.success: + await _mark_agent_attempt_succeeded(attempt, spent_budget=spent_budget) + else: + await _mark_agent_attempt_failed( + attempt, + report.failure_summary or "Server agent did not verify the endpoint", + spent_budget=spent_budget, + ) + + +async def _mark_agent_attempt_succeeded( + attempt: EndpointAgentAttemptModel, + *, + spent_budget: Optional[float], +) -> None: + now = get_current_datetime() + async with get_session_ctx() as session: + stored_attempt = await _get_agent_attempt(session, attempt) + if stored_attempt is None: + return + stored_attempt.status = EndpointAgentAttemptStatus.SUCCEEDED + stored_attempt.status_message = None + if spent_budget is not None: + stored_attempt.spent_agent_budget = spent_budget + stored_attempt.finished_at = stored_attempt.finished_at or now + stored_attempt.updated_at = now + await session.commit() + attempt.status = EndpointAgentAttemptStatus.SUCCEEDED + attempt.status_message = None + attempt.spent_agent_budget = stored_attempt.spent_agent_budget + attempt.finished_at = stored_attempt.finished_at + + +async def _mark_agent_attempt_failed( + attempt: EndpointAgentAttemptModel, + error: str, + *, + spent_budget: Optional[float] = None, +) -> None: + now = get_current_datetime() + async with get_session_ctx() as session: + stored_attempt = await _get_agent_attempt(session, attempt) + if stored_attempt is None: + return + stored_attempt.status = EndpointAgentAttemptStatus.FAILED + stored_attempt.status_message = error + if spent_budget is not None: + stored_attempt.spent_agent_budget = spent_budget + stored_attempt.finished_at = stored_attempt.finished_at or now + stored_attempt.updated_at = now + await session.commit() + attempt.status = EndpointAgentAttemptStatus.FAILED + attempt.status_message = error + attempt.spent_agent_budget = stored_attempt.spent_agent_budget + attempt.finished_at = stored_attempt.finished_at + + +async def _update_agent_attempt_offsets( + attempt: EndpointAgentAttemptModel, + *, + progress_log_offset: Optional[int] = None, + stdout_log_offset: Optional[int] = None, + stderr_log_offset: Optional[int] = None, + spent_budget: Optional[float] = None, +) -> None: + async with get_session_ctx() as session: + stored_attempt = await _get_agent_attempt(session, attempt) + if stored_attempt is None: + return + if progress_log_offset is not None: + stored_attempt.progress_log_offset = progress_log_offset + attempt.progress_log_offset = progress_log_offset + if stdout_log_offset is not None: + stored_attempt.stdout_log_offset = stdout_log_offset + attempt.stdout_log_offset = stdout_log_offset + if stderr_log_offset is not None: + stored_attempt.stderr_log_offset = stderr_log_offset + attempt.stderr_log_offset = stderr_log_offset + if spent_budget is not None: + stored_attempt.spent_agent_budget = spent_budget + attempt.spent_agent_budget = spent_budget + stored_attempt.updated_at = get_current_datetime() + await session.commit() + + +async def _reconcile_agent_progress( + *, + attempt: EndpointAgentAttemptModel, + workspace: "_AgentWorkspace", +) -> None: + offset = await _flush_agent_progress_logs( + workspace, + offset=attempt.progress_log_offset, + include_partial=False, + ) + if offset != attempt.progress_log_offset: + await _update_agent_attempt_offsets(attempt, progress_log_offset=offset) + + +async def _reconcile_agent_stream_files( + *, + attempt: EndpointAgentAttemptModel, + workspace: "_AgentWorkspace", +) -> _AgentProcessOutput: + stdout_output, stdout_offset = await _read_agent_stream_file( + workspace=workspace, + path=workspace.work_dir / _AGENT_STDOUT_LOG_NAME, + offset=attempt.stdout_log_offset, + stream_name="stdout", + ) + stderr_output, stderr_offset = await _read_agent_stream_file( + workspace=workspace, + path=workspace.work_dir / _AGENT_STDERR_LOG_NAME, + offset=attempt.stderr_log_offset, + stream_name="stderr", + ) + output = _merge_process_outputs(stdout_output, stderr_output) + if ( + stdout_offset != attempt.stdout_log_offset + or stderr_offset != attempt.stderr_log_offset + or output.spent_budget is not None + ): + await _update_agent_attempt_offsets( + attempt, + stdout_log_offset=stdout_offset, + stderr_log_offset=stderr_offset, + spent_budget=output.spent_budget, + ) + return output + + +def _load_candidate_run_ids(workspace: "_AgentWorkspace") -> tuple[UUID, ...]: + path = workspace.work_dir / "candidates.jsonl" + if not path.exists(): + return () + run_ids: list[UUID] = [] + seen: set[UUID] = set() + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not line.strip(): + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + _write_trace_record( + workspace, + {"type": "agent-candidate-invalid", "line": line}, + ) + continue + if not isinstance(parsed, dict): + continue + raw_run_id = parsed.get("run_id") + if not isinstance(raw_run_id, str) or not raw_run_id.strip(): + continue + try: + run_id = UUID(raw_run_id) + except ValueError: + _write_trace_record( + workspace, + {"type": "agent-candidate-invalid-run-id", "run_id": raw_run_id}, + ) + continue + if run_id not in seen: + seen.add(run_id) + run_ids.append(run_id) + return tuple(run_ids) + + +def _load_agent_error(workspace: "_AgentWorkspace") -> Optional[str]: + path = workspace.work_dir / "agent_error.json" + if not path.exists(): + return None + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return "Server agent failed" + if not isinstance(parsed, dict): + return "Server agent failed" + failure_summary = parsed.get("failure_summary") + if isinstance(failure_summary, str) and failure_summary.strip(): + return failure_summary + return "Server agent failed" + def get_claude_agent_unavailable_reason() -> Optional[str]: if not settings.AGENT_ANTHROPIC_API_KEY: @@ -258,11 +673,11 @@ async def flush(self) -> None: def _prepare_workspace( *, endpoint_model: EndpointModel, - workspace_base_dir: Path, + workspace_root_dir: Path, ) -> _AgentWorkspace: configuration = EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) - root_dir = workspace_base_dir / str(endpoint_model.id) + root_dir = workspace_root_dir home_dir = root_dir / "home" work_dir = root_dir / "workspace" dstack_dir = home_dir / ".dstack" @@ -662,6 +1077,40 @@ def _format_constraint_value(value: Any) -> str: return str(value) +def _start_agent_subprocess_detached( + workspace: _AgentWorkspace, + request: dict[str, Any], +) -> int: + cmd = _build_claude_command(request) + workspace.artifacts.mark_running() + stdout_path = workspace.work_dir / _AGENT_STDOUT_LOG_NAME + stderr_path = workspace.work_dir / _AGENT_STDERR_LOG_NAME + stdout_path.parent.mkdir(parents=True, exist_ok=True) + with ( + stdout_path.open("ab") as stdout, + stderr_path.open("ab") as stderr, + ): + proc = subprocess.Popen( + cmd, + cwd=workspace.work_dir, + env=request["env"], + stdout=stdout, + stderr=stderr, + close_fds=True, + start_new_session=True, + ) + _write_agent_process_state(workspace, proc.pid) + _write_trace_record( + workspace, + { + "type": "agent-process-started", + "pid": proc.pid, + "host": _get_process_host(), + }, + ) + return proc.pid + + async def _run_agent_in_subprocess( workspace: _AgentWorkspace, request: dict[str, Any], @@ -791,20 +1240,74 @@ async def _read_agent_stream( if not line_bytes: return output line = line_bytes.decode(errors="replace") - output.stdout_tail = _append_bounded_output(output.stdout_tail, line) - if not line.strip(): - continue - try: - message = json.loads(line) - except json.JSONDecodeError: - message = {"type": "raw-output", "stream": stream_name, "line": line.rstrip("\r\n")} - _write_trace_record(workspace, message) - continue - if stream_name != "stdout": - message.setdefault("stream", stream_name) + _process_agent_stream_line( + line=line, + workspace=workspace, + stream_name=stream_name, + output=output, + ) + + +async def _read_agent_stream_file( + *, + workspace: _AgentWorkspace, + path: Path, + offset: int, + stream_name: str, +) -> tuple[_AgentProcessOutput, int]: + output = _AgentProcessOutput() + lines, new_offset = _read_complete_file_lines(path=path, offset=offset) + for line in lines: + _process_agent_stream_line( + line=line, + workspace=workspace, + stream_name=stream_name, + output=output, + ) + return output, new_offset + + +def _process_agent_stream_line( + *, + line: str, + workspace: _AgentWorkspace, + stream_name: str, + output: _AgentProcessOutput, +) -> None: + output.stdout_tail = _append_bounded_output(output.stdout_tail, line) + if not line.strip(): + return + try: + message = json.loads(line) + except json.JSONDecodeError: + message = {"type": "raw-output", "stream": stream_name, "line": line.rstrip("\r\n")} _write_trace_record(workspace, message) - workspace.artifacts.record_stream_message(message) - _update_agent_process_output(output, message) + return + if stream_name != "stdout": + message.setdefault("stream", stream_name) + _write_trace_record(workspace, message) + workspace.artifacts.record_stream_message(message) + _update_agent_process_output(output, message) + + +def _read_complete_file_lines(*, path: Path, offset: int) -> tuple[list[str], int]: + if not path.exists(): + return [], offset + size = path.stat().st_size + if size < offset: + offset = 0 + with path.open("rb") as f: + f.seek(offset) + data = f.read() + if not data: + return [], offset + newline_pos = max(data.rfind(b"\n"), data.rfind(b"\r")) + if newline_pos == -1: + return [], offset + complete = data[: newline_pos + 1] + new_offset = offset + len(complete) + lines = [line.decode(errors="replace") for line in complete.splitlines(keepends=True)] + return lines, new_offset class _AgentProgressLogTailer: @@ -863,6 +1366,50 @@ async def _write_progress_line(self, line: str) -> None: await _write_agent_log(self._workspace, message) +async def _flush_agent_progress_logs( + workspace: _AgentWorkspace, + *, + offset: int, + include_partial: bool, +) -> int: + path = workspace.progress_path + lines, new_offset = _read_complete_file_lines(path=path, offset=offset) + if include_partial and path.exists(): + size = path.stat().st_size + if size > new_offset: + with path.open("rb") as f: + f.seek(new_offset) + data = f.read() + if data: + lines.append(data.decode(errors="replace")) + new_offset = size + for line in lines: + await _write_agent_progress_line(workspace, line.rstrip("\r\n")) + await _flush_agent_logs(workspace) + return new_offset + + +async def _write_agent_progress_line(workspace: _AgentWorkspace, line: str) -> None: + if not line.strip(): + return + try: + parsed = json.loads(line) + except json.JSONDecodeError: + _write_trace_record( + workspace, + {"type": "agent-progress-invalid", "line": line}, + ) + return + message = _format_agent_progress_log_message(parsed) + if message is None: + _write_trace_record( + workspace, + {"type": "agent-progress-ignored", "record": parsed}, + ) + return + await _write_agent_log(workspace, message) + + def _format_agent_progress_log_message(record: Any) -> Optional[str]: if not isinstance(record, dict): return None @@ -881,6 +1428,9 @@ def _update_agent_process_output( ) -> None: if message.get("type") != "result": return + total_cost = message.get("total_cost_usd") + if isinstance(total_cost, (int, float)): + output.spent_budget = float(total_cost) if message.get("is_error"): output.result_error = message.get("result") or "Claude agent failed" structured_output = message.get("structured_output") @@ -906,6 +1456,8 @@ def _merge_process_outputs(*outputs: _AgentProcessOutput) -> _AgentProcessOutput merged.report_data = output.report_data if output.result_error is not None: merged.result_error = output.result_error + if output.spent_budget is not None: + merged.spent_budget = output.spent_budget merged.stdout_tail = _append_bounded_output(merged.stdout_tail, output.stdout_tail) return merged @@ -975,6 +1527,7 @@ def _write_agent_process_state(workspace: _AgentWorkspace, pid: int) -> None: path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME data = { "pid": pid, + "host": _get_process_host(), "started_at": _utcnow_iso(), } path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") @@ -993,15 +1546,28 @@ def _clear_agent_process_state(workspace: _AgentWorkspace, pid: int) -> None: path.unlink(missing_ok=True) -def _get_running_agent_process_pid(workspace: _AgentWorkspace) -> Optional[int]: +def _get_running_agent_process_pid( + workspace: _AgentWorkspace, + attempt: Optional[EndpointAgentAttemptModel] = None, +) -> Optional[int]: + if attempt is not None and attempt.process_host not in (None, _get_process_host()): + return attempt.pid path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME if not path.exists(): - return None + if attempt is None or attempt.pid is None: + return None + if attempt.process_host not in (None, _get_process_host()): + return attempt.pid + return attempt.pid if _is_process_running(attempt.pid) else None try: data = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError: path.unlink(missing_ok=True) return None + host = data.get("host") + if isinstance(host, str) and host and host != _get_process_host(): + pid = data.get("pid") + return pid if isinstance(pid, int) else None pid = data.get("pid") if not isinstance(pid, int): path.unlink(missing_ok=True) @@ -1012,6 +1578,10 @@ def _get_running_agent_process_pid(workspace: _AgentWorkspace) -> Optional[int]: return None +def _get_process_host() -> str: + return socket.gethostname() + + def _is_process_running(pid: int) -> bool: try: os.kill(pid, 0) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index cc75fe473a..68bc3b679d 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -22,6 +22,7 @@ Resources, ) from dstack._internal.core.models.runs import JobStatus, RunSpec, RunStatus, ServiceSpec +from dstack._internal.core.models.users import GlobalRole, ProjectRole from dstack._internal.server.background.pipeline_tasks.endpoints import ( EndpointPipelineItem, EndpointWorker, @@ -33,6 +34,7 @@ from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport from dstack._internal.server.services.endpoints.names import get_endpoint_serving_run_name from dstack._internal.server.services.endpoints.planning import EndpointPresetPlanningResult +from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( create_job, create_project, @@ -86,9 +88,22 @@ async def _create_endpoint_model( session: AsyncSession, status: EndpointStatus = EndpointStatus.SUBMITTED, user_ssh_public_key: str | None = None, + user_global_role: GlobalRole = GlobalRole.ADMIN, + project_role: ProjectRole | None = None, ) -> EndpointModel: project = await create_project(session=session) - user = await create_user(session=session, ssh_public_key=user_ssh_public_key) + user = await create_user( + session=session, + global_role=user_global_role, + ssh_public_key=user_ssh_public_key, + ) + if project_role is not None: + await add_project_member( + session=session, + project=project, + user=user, + project_role=project_role, + ) configuration = EndpointConfiguration( name="qwen-endpoint", model="Qwen/Qwen3-0.6B", @@ -656,6 +671,39 @@ async def test_submitted_to_clauding_with_agent( assert len(events) == 1 assert events[0].message == "Endpoint status changed SUBMITTED -> CLAUDING" + async def test_project_user_cannot_start_agent( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + user_global_role=GlobalRole.USER, + project_role=ProjectRole.USER, + ) + agent_service = _FakeAgentService() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "No matching endpoint presets found. " + "Creating endpoint presets with the server agent requires project admin permissions." + ) + assert endpoint_model.service_run_id is None + assert endpoint_model.provisioning_method is None + agent_service.provision_endpoint.assert_not_awaited() + async def test_submitted_to_provisioning_with_matching_preset( self, test_db, session: AsyncSession, worker: EndpointWorker ): @@ -922,6 +970,76 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): agent_service.provision_endpoint.assert_awaited_once() preset_service.save_preset.assert_awaited_once() + async def test_agent_success_after_stop_keeps_endpoint_stopping_and_links_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.CLAUDING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + created_run_id = None + + async def provision_endpoint(endpoint_model, pipeline_hinter): + nonlocal created_run_id + async with get_session_ctx() as provisioning_session: + res = await provisioning_session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_model.id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + provisioning_endpoint = res.unique().scalar_one() + run = await _create_backing_service_run( + session=provisioning_session, + endpoint_model=provisioning_endpoint, + link_endpoint=False, + run_name=_get_agent_run_name(), + ) + provisioning_endpoint.status = EndpointStatus.STOPPING + await provisioning_session.commit() + created_run_id = run.id + return _get_verified_agent_result(run) + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + preset_service = Mock() + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + assert created_run_id is not None + await session.refresh(endpoint_model) + run = await session.get(RunModel, created_run_id) + assert run is not None + assert endpoint_model.status == EndpointStatus.STOPPING + assert endpoint_model.service_run_id == run.id + assert run.status == RunStatus.RUNNING + assert preset_service.save_preset.await_count == 1 + events = await list_events(session) + assert events == [] + + await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.STOPPING + assert run.status == RunStatus.TERMINATING + async def test_agent_reported_run_id_links_backing_service_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): @@ -1096,6 +1214,45 @@ async def test_agent_in_progress_keeps_endpoint_clauding( assert endpoint_model.service_run_id is None agent_service.provision_endpoint.assert_awaited_once() + async def test_agent_in_progress_records_candidate_run_submission( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.CLAUDING, + ) + endpoint_model.provisioning_method = "agent" + candidate_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=_get_agent_run_name("1"), + ) + endpoint_model.service_run_id = None + await session.commit() + agent_service = _FakeAgentService( + result=AgentProvisioningResult( + in_progress=True, + candidate_run_ids=(candidate_run.id,), + ) + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.service_run_id is None + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert len(submissions) == 1 + assert submissions[0].run_id == candidate_run.id + async def test_agent_error_status_message_is_compact( self, test_db, session: AsyncSession, worker: EndpointWorker ): @@ -1206,6 +1363,48 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): assert run.status == RunStatus.RUNNING agent_service.provision_endpoint.assert_awaited_once() + async def test_agent_failure_stops_recorded_candidate_run( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.CLAUDING, + ) + endpoint_model.provisioning_method = "agent" + candidate_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=_get_agent_run_name("1"), + ) + endpoint_model.service_run_id = None + await session.commit() + agent_service = _FakeAgentService( + result=AgentProvisioningResult( + error="agent could not verify the service", + candidate_run_ids=(candidate_run.id,), + ) + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(candidate_run) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == "agent could not verify the service" + assert endpoint_model.service_run_id is None + assert candidate_run.status == RunStatus.TERMINATING + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert len(submissions) == 1 + assert submissions[0].run_id == candidate_run.id + async def test_waits_when_backing_run_is_not_ready( self, test_db, session: AsyncSession, worker: EndpointWorker ): @@ -1520,6 +1719,65 @@ async def test_stops_backing_run_before_stopping_endpoint( assert run.status == RunStatus.TERMINATING assert run.deleted is False + async def test_stops_recorded_candidate_run_before_stopping_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.STOPPING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.RUNNING, + link_endpoint=False, + run_name=_get_agent_run_name("1"), + ) + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run.id, + ) + await session.commit() + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.STOPPING + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.TERMINATING + + async def test_waits_for_terminating_recorded_candidate_run_before_stopping_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.STOPPING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + status=RunStatus.TERMINATING, + link_endpoint=False, + run_name=_get_agent_run_name("1"), + ) + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run.id, + ) + await session.commit() + + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(run) + assert endpoint_model.status == EndpointStatus.STOPPING + assert run.status == RunStatus.TERMINATING + events = await list_events(session) + assert events == [] + async def test_marks_endpoint_stopped_after_backing_run_finishes( self, test_db, session: AsyncSession, worker: EndpointWorker ): diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index b6058541b7..918e042e20 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -139,7 +139,7 @@ async def test_returns_agent_provisioning_plan( user = await create_user(session, global_role=GlobalRole.USER) project = await create_project(session) await add_project_member( - session=session, project=project, user=user, project_role=ProjectRole.USER + session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) agent_service = Mock() agent_service.is_enabled.return_value = True @@ -179,6 +179,51 @@ async def test_returns_agent_provisioning_plan( "reason": None, } + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_no_agent_plan_for_project_user( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + agent_service = Mock() + agent_service.is_enabled.return_value = True + + with ( + patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.services.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + }, + }, + ) + + assert response.status_code == 200, response.json() + assert response.json()["provisioning_plan"] == { + "type": "none", + "reason": ( + "No matching endpoint presets found. " + "Creating endpoint presets with the server agent requires project admin " + "permissions." + ), + } + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_returns_preset_provisioning_plan( @@ -236,7 +281,7 @@ async def test_creates_endpoint(self, test_db, session: AsyncSession, client: As user = await create_user(session, global_role=GlobalRole.USER) project = await create_project(session) await add_project_member( - session=session, project=project, user=user, project_role=ProjectRole.USER + session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) response = await client.post( @@ -282,7 +327,7 @@ async def test_resubmits_terminal_endpoint( user = await create_user(session, global_role=GlobalRole.USER) project = await create_project(session) await add_project_member( - session=session, project=project, user=user, project_role=ProjectRole.USER + session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) previous_endpoint = await _create_endpoint_model( session=session, @@ -318,7 +363,7 @@ async def test_resubmits_terminal_endpoint( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - async def test_rejects_duplicate_non_terminal_endpoint( + async def test_rejects_project_user_when_create_requires_agent( self, test_db, session: AsyncSession, client: AsyncClient ): user = await create_user(session, global_role=GlobalRole.USER) @@ -326,6 +371,70 @@ async def test_rejects_duplicate_non_terminal_endpoint( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) + + with patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + } + }, + ) + + assert response.status_code == 403 + assert response.json()["msg"] == ( + "Creating endpoint presets with the server agent requires project admin permissions." + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_allows_project_user_when_matching_preset_exists( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.USER + ) + + with patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock( + return_value=EndpointPresetPlanningResult(provisionable=_endpoint_preset_plan()) + ), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + } + }, + ) + + assert response.status_code == 200, response.json() + assert response.json()["status"] == "submitted" + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_rejects_duplicate_non_terminal_endpoint( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) await _create_endpoint_model( session=session, project=project, diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index c583c4bf39..dad7451562 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -5,6 +5,7 @@ import pytest import yaml +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession import dstack._internal.server.services.endpoints.agent.claude as claude_module @@ -13,7 +14,11 @@ from dstack._internal.core.models.envs import Env from dstack._internal.core.models.profiles import SpotPolicy from dstack._internal.server import settings -from dstack._internal.server.models import EndpointModel +from dstack._internal.server.models import ( + EndpointAgentAttemptModel, + EndpointAgentAttemptStatus, + EndpointModel, +) from dstack._internal.server.services.endpoints.agent.claude import ( ClaudeAgentService, _AgentRunnerResult, @@ -73,6 +78,46 @@ def _configure_fake_claude(tmp_path, monkeypatch: pytest.MonkeyPatch) -> str: class TestClaudeAgentService: + @pytest.mark.asyncio + async def test_default_runner_starts_agent_detached( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") + started = {} + + def start_agent(workspace, request): + started["workspace"] = workspace + started["request"] = request + return 12345 + + monkeypatch.setattr(claude_module, "_start_agent_subprocess_detached", start_agent) + endpoint_model = await _create_endpoint_model(session, max_agent_budget=1.5) + service = ClaudeAgentService(workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.in_progress is True + assert result.error is None + attempt_root = tmp_path / str(endpoint_model.id) / "1" + assert started["workspace"].root_dir == attempt_root + assert started["request"]["cwd"] == str(attempt_root / "workspace") + assert started["request"]["options"]["max_budget"] == 1.5 + res = await session.execute( + select(EndpointAgentAttemptModel).where( + EndpointAgentAttemptModel.endpoint_id == endpoint_model.id + ) + ) + attempt = res.scalar_one() + assert attempt.attempt_num == 1 + assert attempt.status == EndpointAgentAttemptStatus.RUNNING + assert attempt.pid == 12345 + assert attempt.workspace_path == str(attempt_root) + @pytest.mark.asyncio async def test_invokes_agent_with_isolated_dstack_cli_context( self, @@ -116,10 +161,11 @@ async def runner(workspace, request): "successful final report must include the verified service run `run_id`" in captured["request"]["prompt"] ) - assert captured["request"]["cwd"] == str(tmp_path / str(endpoint_model.id) / "workspace") + attempt_root = tmp_path / str(endpoint_model.id) / "1" + assert captured["request"]["cwd"] == str(attempt_root / "workspace") env = captured["request"]["env"] assert env["ANTHROPIC_API_KEY"] == "agent-secret" - assert env["HOME"] == str(tmp_path / str(endpoint_model.id) / "home") + assert env["HOME"] == str(attempt_root / "home") assert env["DSTACK_PROJECT"] == "main" assert env["HF_TOKEN"] == "hf-secret" assert "DATABASE_URL" not in env @@ -138,9 +184,7 @@ async def runner(workspace, request): assert command[command.index("--permission-mode") + 1] == "bypassPermissions" assert "--max-budget-usd" in command assert command[command.index("--max-budget-usd") + 1] == "1.5" - config = yaml.safe_load( - (tmp_path / str(endpoint_model.id) / "home" / ".dstack" / "config.yml").read_text() - ) + config = yaml.safe_load((attempt_root / "home" / ".dstack" / "config.yml").read_text()) assert config == { "projects": [ { @@ -151,7 +195,7 @@ async def runner(workspace, request): } ], } - work_dir = tmp_path / str(endpoint_model.id) / "workspace" + work_dir = attempt_root / "workspace" state = json.loads((work_dir / "agent_state.json").read_text()) assert state["endpoint_name"] == "qwen-endpoint" assert state["model"] == "Qwen/Qwen3-0.6B" @@ -271,7 +315,7 @@ async def runner(workspace, request): in prompt ) assert "RUNPOD" not in prompt - work_dir = tmp_path / str(endpoint_model.id) / "workspace" + work_dir = tmp_path / str(endpoint_model.id) / "1" / "workspace" prototyping_skill = ( work_dir / ".claude" / "skills" / "dstack-prototyping" / "SKILL.md" ).read_text() @@ -354,10 +398,11 @@ async def test_reuses_existing_final_report_without_invoking_runner( encoding="utf-8", ) - async def runner(workspace, request): - raise AssertionError("runner must not be invoked") + def start_agent(workspace, request): + raise AssertionError("agent process must not be started") - service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + monkeypatch.setattr(claude_module, "_start_agent_subprocess_detached", start_agent) + service = ClaudeAgentService(workspace_base_dir=tmp_path) result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) @@ -367,6 +412,94 @@ async def runner(workspace, request): assert result.final_report is not None assert result.final_report.verification_summary == "Verified before restart." + @pytest.mark.asyncio + async def test_waits_for_claude_result_after_final_report_while_process_runs( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr( + claude_module, + "_get_running_agent_process_pid", + lambda workspace, attempt=None: 123, + ) + endpoint_model = await _create_endpoint_model(session) + work_dir = tmp_path / str(endpoint_model.id) / "workspace" + work_dir.mkdir(parents=True) + (work_dir / "final_report.json").write_text( + json.dumps( + { + "success": True, + "run_id": str(uuid.uuid4()), + "run_name": "qwen-agent-candidate", + "service_yaml": "type: service\nname: qwen-agent-candidate\n", + "verification_summary": "Verified before Claude exited.", + } + ), + encoding="utf-8", + ) + + service = ClaudeAgentService(workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.in_progress is True + assert result.final_report is None + + @pytest.mark.asyncio + async def test_reconciles_claude_cost_from_result_stream( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + endpoint_model = await _create_endpoint_model(session) + work_dir = tmp_path / str(endpoint_model.id) / "workspace" + work_dir.mkdir(parents=True) + run_id = uuid.uuid4() + (work_dir / "final_report.json").write_text( + json.dumps( + { + "success": True, + "run_id": str(run_id), + "run_name": "qwen-agent-candidate", + "service_yaml": "type: service\nname: qwen-agent-candidate\n", + "verification_summary": "Verified before restart.", + } + ), + encoding="utf-8", + ) + (work_dir / "agent_stdout.jsonl").write_text( + json.dumps( + { + "type": "result", + "is_error": False, + "total_cost_usd": 0.42, + } + ) + + "\n", + encoding="utf-8", + ) + service = ClaudeAgentService(workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + assert result.run_id == run_id + res = await session.execute( + select(EndpointAgentAttemptModel).where( + EndpointAgentAttemptModel.endpoint_id == endpoint_model.id + ) + ) + attempt = res.scalar_one() + assert attempt.status == EndpointAgentAttemptStatus.SUCCEEDED + assert attempt.spent_agent_budget == 0.42 + @pytest.mark.asyncio async def test_returns_in_progress_when_agent_process_is_still_running( self, @@ -376,13 +509,18 @@ async def test_returns_in_progress_when_agent_process_is_still_running( ): monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") _configure_fake_claude(tmp_path, monkeypatch) - monkeypatch.setattr(claude_module, "_get_running_agent_process_pid", lambda workspace: 123) + monkeypatch.setattr( + claude_module, + "_get_running_agent_process_pid", + lambda workspace, attempt=None: 123, + ) endpoint_model = await _create_endpoint_model(session) - async def runner(workspace, request): - raise AssertionError("runner must not be invoked") + def start_agent(workspace, request): + raise AssertionError("agent process must not be started") - service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + monkeypatch.setattr(claude_module, "_start_agent_subprocess_detached", start_agent) + service = ClaudeAgentService(workspace_base_dir=tmp_path) result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) @@ -423,7 +561,7 @@ async def runner(workspace, request): result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) assert result.error is None - trace_path = tmp_path / str(endpoint_model.id) / "trace.jsonl" + trace_path = tmp_path / str(endpoint_model.id) / "1" / "trace.jsonl" trace = trace_path.read_text() assert "agent-secret" not in trace assert "hf-secret" not in trace @@ -455,7 +593,7 @@ async def runner(workspace, request): result.error == "Server agent failed before returning a verification report: bad key" ) assert result.final_report is None - work_dir = tmp_path / str(endpoint_model.id) / "workspace" + work_dir = tmp_path / str(endpoint_model.id) / "1" / "workspace" state = json.loads((work_dir / "agent_state.json").read_text()) assert state["phase"] == "failure" agent_error = json.loads((work_dir / "agent_error.json").read_text()) From 19bdaf0029afa55c0e64d6b30e1bd1204a3adafc Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 8 Jul 2026 15:09:14 +0200 Subject: [PATCH 16/21] Checkpoint endpoint agent draft --- endpoint-agent-checkpoints.md | 340 ++++- endpoint-agent-harness-test-plan.md | 79 +- endpoint-e2e-testing-report.md | 1233 ++++++++++++++++- endpoint-implementation-plan.md | 442 +++--- .../dstack-prototyping-skill-audit.md | 115 ++ .../dstack-prototyping-skill-outline.md | 11 + skills/dstack-prototyping/SKILL.md | 269 +--- src/dstack/_internal/cli/commands/endpoint.py | 83 +- .../_internal/cli/services/completion.py | 2 +- .../cli/services/configurators/endpoint.py | 17 +- .../_internal/cli/services/endpoint_logs.py | 76 + src/dstack/_internal/cli/utils/endpoint.py | 28 +- src/dstack/_internal/cli/utils/preset.py | 93 +- .../_internal/core/models/endpoint_presets.py | 27 +- src/dstack/_internal/core/models/endpoints.py | 30 +- .../background/pipeline_tasks/endpoints.py | 257 +++- ...1d6c2a9b77_add_endpoint_agent_sessions.py} | 30 +- src/dstack/_internal/server/models.py | 18 +- .../_internal/server/routers/endpoints.py | 12 +- .../server/schemas/endpoint_presets.py | 4 +- .../server/services/endpoints/__init__.py | 143 +- .../services/endpoints/agent/__init__.py | 23 +- .../server/services/endpoints/agent/claude.py | 1042 +++++++++----- .../server/services/endpoints/agent/report.py | 8 +- .../agent/resources/system_prompt.md | 198 ++- .../server/services/endpoints/planning.py | 66 +- .../services/endpoints/preset_building.py | 37 +- .../server/services/endpoints/presets.py | 724 ++++++---- src/dstack/_internal/server/settings.py | 10 - src/dstack/api/server/_endpoint_presets.py | 12 +- .../_internal/cli/commands/test_endpoint.py | 59 +- .../services/configurators/test_endpoint.py | 27 +- .../_internal/cli/utils/test_endpoint.py | 41 +- src/tests/_internal/cli/utils/test_preset.py | 216 +-- .../_internal/core/models/test_endpoints.py | 14 - .../pipeline_tasks/test_endpoints.py | 447 ++++-- .../server/routers/test_endpoints.py | 318 ++++- .../services/endpoints/test_agent_report.py | 17 +- .../services/endpoints/test_claude_agent.py | 704 ++++++---- .../server/services/test_endpoint_presets.py | 835 ++++++----- 40 files changed, 5876 insertions(+), 2231 deletions(-) create mode 100644 endpoint-implementation-research/dstack-prototyping-skill-audit.md create mode 100644 endpoint-implementation-research/dstack-prototyping-skill-outline.md create mode 100644 src/dstack/_internal/cli/services/endpoint_logs.py rename src/dstack/_internal/server/migrations/versions/2026/{07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py => 07_06_1700_4e1d6c2a9b77_add_endpoint_agent_sessions.py} (67%) diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md index cf9675c976..40f2c6ff66 100644 --- a/endpoint-agent-checkpoints.md +++ b/endpoint-agent-checkpoints.md @@ -227,9 +227,9 @@ Observed result: ### Endpoint Preset CLI -`dstack preset` now lists endpoint presets saved on the server. `dstack preset list` -is equivalent, and `dstack preset delete NAME` removes a saved endpoint preset after -confirmation. +`dstack endpoint preset` lists endpoint presets saved on the server. +`dstack endpoint preset get MODEL --json` returns the model-level preset, and +`dstack endpoint preset delete MODEL` removes it after confirmation. This is intentionally limited to list/delete. Creating or updating presets remains part of the endpoint agent flow, where the agent saves a preset only after verifying the final @@ -240,9 +240,9 @@ Verification on 2026-07-04: ```bash uv run pytest src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/core/models/endpoint_presets.py src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/schemas/endpoint_presets.py src/dstack/api/server/_endpoint_presets.py src/dstack/api/server/__init__.py src/dstack/_internal/server/routers/endpoints.py src/dstack/_internal/cli/utils/preset.py src/dstack/_internal/cli/commands/preset.py src/dstack/_internal/cli/services/completion.py src/dstack/_internal/cli/main.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py -uv run dstack preset --help -uv run dstack preset +uv run ruff check src/dstack/_internal/core/models/endpoint_presets.py src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/schemas/endpoint_presets.py src/dstack/api/server/_endpoint_presets.py src/dstack/api/server/__init__.py src/dstack/_internal/server/routers/endpoints.py src/dstack/_internal/cli/utils/preset.py src/dstack/_internal/cli/commands/endpoint.py src/dstack/_internal/cli/services/completion.py src/dstack/_internal/cli/main.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py +uv run dstack endpoint preset --help +uv run dstack endpoint preset uv run dstack --help ``` @@ -251,17 +251,17 @@ Observed result: - focused preset/router pytest: `38 passed, 11 skipped` - broader endpoint/preset pytest: `125 passed, 46 skipped` - ruff: `All checks passed!` -- `dstack preset` lists the saved Qwen endpoint presets -- top-level help now includes `preset Manage endpoint presets` +- `dstack endpoint preset` lists the saved Qwen endpoint presets +- endpoint help includes `preset Manage endpoint presets` ### Endpoint Preset Resource Contract Endpoint presets now separate scheduling requirements from verified runtime evidence: -`replica_spec_groups[*].resources` is used for service planning and offer matching, -while `replica_spec_groups[*].tested_resources` stores exact resources captured from -actual registered service replicas. +recipe `service.resources` is used for service planning and offer matching, while +`validations[*].replicas[*].resources` stores exact resources captured from actual +registered service replicas. -`dstack preset` now displays every actual replica when a preset has multiple replicas, using child rows such as `replica=0` or `group=worker replica=1`, matching the hierarchy used by `dstack ps`. It does not summarize replicas as counts. +`dstack endpoint preset` now displays one row per recipe, and expands service replica groups with child rows when a recipe has multiple groups. Exact validation resources stay in `get --json`. Invalid local preset files are skipped for user-facing preset listing and logged server-side with the preset path and parse/validation error. @@ -272,7 +272,7 @@ Verification on 2026-07-04: uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/cli/utils/test_preset.py uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py uv run ruff check src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/services/endpoints/preset_building.py src/dstack/_internal/cli/utils/preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/cli/utils/test_preset.py -uv run dstack preset +uv run dstack endpoint preset ``` Observed result: @@ -280,7 +280,7 @@ Observed result: - focused preset/endpoint-worker pytest: `69 passed, 35 skipped` - broader endpoint pytest: `138 passed, 46 skipped` - ruff: `All checks passed!` -- `dstack preset` skips the old loose smoke preset and lists the valid learned preset; the server logs the skipped preset path and validation error +- `dstack endpoint preset` skips the old loose smoke preset and lists the valid learned preset; the server logs the skipped preset path and validation error ### Endpoint Agent Retest After Deleting Preset @@ -310,7 +310,7 @@ Important harness findings: requirements (`gpu.name: [L4, A40, RTX3090]`) instead of preserving the broadest correct model-derived requirement. - Bad: the agent's verification/final report said L4, but the actual provisioned - hardware and saved `tested_resources` were A40. The server-side preset builder used + hardware and saved validation resources were A40. The server-side preset builder used actual run state correctly; the agent report was stale/inferred. Patch made after this run: @@ -363,7 +363,7 @@ Default CLI project preserved: `main -> http://127.0.0.1:3002` ```bash uv run dstack endpoint --project endpoint-dev get qwen-endpoint-smoke --json uv run dstack run --project endpoint-dev get qwen-smoke --json -uv run dstack preset --project endpoint-dev +uv run dstack endpoint --project endpoint-dev preset uv run dstack logs --project endpoint-dev qwen-smoke --since 3m ``` @@ -371,7 +371,7 @@ Observed result on 2026-07-04: - endpoint status: `running` - backing service status: `running` -- preset listed by `dstack preset --project endpoint-dev` +- preset listed by `dstack endpoint --project endpoint-dev preset` - `verification.json` recorded HTTP 200 for `/v1/models` and `/v1/chat/completions` - `final_report.json` recorded actual provisioned hardware from run JSON @@ -389,7 +389,7 @@ These are outside the repo and are not part of the checkpoint commit: ### Known Issues / Next Hardening -- After agent verification, the endpoint briefly transitions from `agenting` to +- After agent verification, the endpoint briefly transitions from `prototyping` to `provisioning` before `running`. This is confusing; the next patch should avoid exposing that intermediate status for agent-verified endpoints. - The agent still sometimes uses invalid CLI forms first, such as `dstack run list` @@ -418,9 +418,9 @@ Status behavior: - Agent-created endpoints no longer expose an intermediate `provisioning` state after the agent returns a verified service run. - If the reported service is not yet fully visible as a ready dstack service, the - endpoint stays `agenting` with `service_run_id` linked. + endpoint stays `prototyping` with `service_run_id` linked. - Once the linked service is ready, the worker saves the learned preset and moves the - endpoint directly from `agenting` to `running`. + endpoint directly from `prototyping` to `running`. - Preset-based endpoint creation still uses `provisioning`. Endpoint log behavior: @@ -450,3 +450,303 @@ Observed result: - broader endpoint/log pytest: `104 passed, 11 skipped` - ruff: `All checks passed!` - format check: `4 files already formatted` + +## Checkpoint: qwen2-runpod-restart-resource-envelope + +Status: known-good same-host restart plus learned-preset resource-envelope validation. + +Suggested local tag after the next checkpoint commit: + +```bash +endpoint-agent/qwen2-runpod-restart-resource-envelope +``` + +Date: 2026-07-07 +Project: `endpoint-e2e` +Server: current checkout on `127.0.0.1:3000` +Test directory: `/Users/dstack/dstack-endpoints-demo/endpoint-agent-restart` + +### What Worked + +- Endpoint `qwen2-05b-restart-1110` reached `running`. +- Model: `Qwen/Qwen2-0.5B-Instruct`. +- Server restart during `prototyping` reused the same agent session/workspace and did not start a + duplicate Claude process. +- Claude submitted and verified service run `qwen2-05b-restart-1110-1`. +- Final run ID: `c1be5e1a-6c49-4ef5-926e-4d2171a03d98`. +- Backend/hardware: RunPod `EU-CZ-1`, NVIDIA RTX 3090 24GB. +- Hourly price: `$0.46/hr`; final run cost after cleanup: `$0.0484`. +- Claude reported agent cost: `$1.1923`. +- Agent verified `/v1/chat/completions` through the dstack proxy with HTTP 200 and response model + `Qwen/Qwen2-0.5B-Instruct`. +- Endpoint preset was saved as `qwen-qwen2-0-5b-instruct-c1be5e1a`. +- Final service YAML used scheduling resources `gpu: 16GB..24GB:1`, not bare `gpu: 1` and not exact + RTX 3090 resources. +- Saved preset kept reusable scheduling resources separate from exact tested hardware: + - scheduling: `cpu=2.. mem=8GB.. disk=30GB.. gpu=16GB..24GB:1` + - tested: `cpu=32 mem=125GB disk=100GB gpu=RTX3090:24GB:1` +- `dstack endpoint stop qwen2-05b-restart-1110 -y` stopped the endpoint and terminated the backing + service run. +- A no-cost reuse preview for the same model found offers without Claude, but selected the older + duplicate preset `qwen-qwen2-0-5b-instruct-532ddf05`, not this checkpoint's new + `qwen-qwen2-0-5b-instruct-c1be5e1a` preset. + +### Useful Runtime Artifacts + +- Workspace: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace` +- Final report: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace/final_report.json` +- Verification: + `/Users/dstack/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace/verification.json` + +### Known Issues / Next Hardening + +- Claude first wrote `backend` instead of `backends` and omitted `fleets`; the workspace CLI guard + caught both before paid submission, but the prompt/skill should reduce this wasted turn. +- Endpoint logs showed Claude assistant stdout even though `progress.jsonl` was clean. Fixed after + this run: assistant stream text is now trace-only; endpoint logs use `progress.jsonl` and explicit + server lifecycle messages. +- The run recorded successful CUDA/vLLM/FlashAttention behavior but did not capture direct + `nvidia-smi` host driver output. +- Duplicate presets for the same model now exist from repeated tests. Keep that inspectable in v1; + automatic update/repair policy is later, but selection/ranking needs a v1 decision if duplicates + can change which preset is reused. + +### Verification Commands + +```bash +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py +uv run pytest src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +uv run ruff check . +uv run pyright -p . +``` + +Observed result after the endpoint-log fix: + +- focused agent pytest: `30 passed` +- broader endpoint pytest slice: `124 passed, 72 skipped` +- ruff: `All checks passed!` +- pyright: `0 errors, 0 warnings, 0 informations` + +## Checkpoint: qwen25-7b-backend-placement-task-first + +Status: known-good backend-placement/task-first validation, with a probe-quality fix applied after +review. + +Suggested local tag after the next checkpoint commit: + +```bash +endpoint-agent/qwen25-7b-backend-placement-task-first +``` + +Date: 2026-07-07 +Project: `endpoint-agent-reasoning` +Endpoint: `qwen25-7b-placement-choice` +Model: `Qwen/Qwen2.5-7B-Instruct` +Fleet: `reusable-vs-container` + +### What Worked + +- The allowed fleet exposed cheaper RunPod container offers and a JarvisLabs L4 reusable/inspectable + offer. +- Claude chose JarvisLabs L4 at `$0.44/hr` over cheaper RunPod offers because it could reuse/inspect + the instance and keep a warm path for the final service. +- Claude submitted a task first: `qwen25-7b-placement-choice-1`. +- Claude then submitted the final service: `qwen25-7b-placement-choice-2`. +- The service reused the warm JarvisLabs instance and verified: + - `/v1/models=200` + - `/v1/chat/completions=200` + - response model matched `Qwen/Qwen2.5-7B-Instruct` +- Endpoint reached `running`, saved a preset, then was stopped cleanly. +- Temporary fleet was deleted afterward; `dstack fleet --project endpoint-agent-reasoning` showed no + fleets. + +### What Was Not Proven + +- The agent did not SSH into the task and did not use a dev environment. +- This proves task-first on reusable backend capacity, not an interactive SSH/dev loop. +- The task probe was too shallow: it observed `nvidia-smi` and driver evidence, but failed on + `python` missing before proving framework import/runtime/server behavior. + +### Fix Applied After Review + +- Endpoint system prompt and `skills/dstack-prototyping/SKILL.md` now say that `nvidia-smi` is host + evidence only. +- A task/dev probe must exercise the intended serving image/runtime/command before it can justify + promotion. +- Final service verification remains the success gate; if final service verification fails, the + agent must return to the evidence loop instead of writing success. + +### Verification Commands + +```bash +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +``` + +Observed result: + +- focused agent pytest: `31 passed` +- broader endpoint/preset/pipeline pytest: `117 passed, 50 skipped` + +## Checkpoint: endpoint-probe-task-shape-guard + +Status: structural harness fix after the batch-task failure. + +Date: 2026-07-07 + +### Why + +The previous prompt/skill change was not enough by itself. In the next live run, Claude added an +optional Hugging Face cache mount, but still encoded the whole probe as a batch task and chose a +RunPod service-like path. That means the agent understood part of the instruction while still +missing the core shape: a probe task should usually be a live environment the agent can attach/SSH +into, not a one-shot shell script. + +### Fix + +- The endpoint agent's local `dstack` wrapper now rejects batch-style endpoint probe tasks. +- Allowed task probe shape: a single long-lived idle command such as `sleep infinity`, with checks + run later through attach/SSH. +- Rejected task probe shape: commands that pack `nvidia-smi`, Python/framework imports, `vllm` / + `sglang`, server startup, or `curl` probes into the task YAML. +- Services are unaffected; the final endpoint proof is still a verified dstack service. + +### Verification Commands + +```bash +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +uv run ruff check src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +``` + +Observed result: + +- focused agent pytest: `33 passed` +- broader endpoint/preset/pipeline pytest: `119 passed, 50 skipped` +- ruff: `All checks passed` + +## Checkpoint: qwen25-probe-quality-cached-batch-abort + +Status: aborted validation run; confirms cache-mount prompt worked but interactive-probe prompt did +not. + +Date: 2026-07-07 +Project: `endpoint-agent-reasoning` +Endpoint: `qwen25-probe-quality-2202` +Fleet: `probe-quality-mixed` + +### What Improved + +- Generated task YAML included an optional Hugging Face instance cache mount: + `/dstack-cache/huggingface` → `/root/.cache/huggingface`, `optional: true`. +- Planned checks still covered vLLM import, local server start, health, and chat completion. + +### What Was Still Wrong + +- The probe remained a batch `commands` chain instead of a long-lived task plus attach/SSH + inspection. +- Claude again wrote `jarvislabs+runpod (container-style)` without stronger backend evidence. +- The actual submitted run landed on RunPod L4, so the placement preference regressed. + +### Cleanup + +- Endpoint `qwen25-probe-quality-2202`: `stopped` +- Task `qwen25-probe-quality-2202-1`: `terminated`, `termination_reason=terminated_by_user` +- Temporary fleet `probe-quality-mixed`: deleted + +### Lesson + +Prompt-only enforcement is not enough here. The next useful change should give the agent explicit +server-generated backend/fleet capability context and should consider a pre-submit guard or required +artifact for interactive probes, instead of adding more generic prose. + +## Checkpoint: qwen25-probe-quality-service-first-abort + +Status: aborted validation run; useful harness failure, not a passing endpoint e2e. + +Date: 2026-07-07 +Project: `endpoint-agent-reasoning` +Endpoint: `qwen25-probe-quality-2136` +Fleet: `probe-quality-mixed` + +### What Happened + +- Temporary fleet exposed RunPod A5000/L4/RTX3090 offers and JarvisLabs L4. +- Claude wrote that both RunPod and JarvisLabs were "container-only (no reusable VM/SSH state)". +- Claude skipped the task/dev probe and submitted direct service `qwen25-probe-quality-2136-1`. +- The service landed on RunPod CA-MTL-1 A5000 at `$0.27/hr`. +- We stopped the endpoint because the run no longer tested the probe-quality fix. + +### Cleanup + +- Endpoint `qwen25-probe-quality-2136`: `stopped` +- Service `qwen25-probe-quality-2136-1`: `terminated`, `termination_reason=terminated_by_user` +- Temporary fleet `probe-quality-mixed`: deleted + +### Lesson + +The failure happened before probe quality. The agent inferred backend capability from the offer +table, turned uncertainty into "no reusable state", and then optimized back to the cheaper RunPod +service-first path. + +Prompt/skill correction after this run: + +- Do not infer "container-only" or "no reusable state" from offers alone. +- Fleet state (`nodes: 0..N`, `idle_duration`, idle/running instances) matters. +- If backend reuse/SSH/cache behavior is uncertain, resolve that uncertainty; do not use it as a + reason to skip a task/dev probe. +- Tiny/well-known model is not enough by itself to skip a probe for a create-recipe endpoint on an + unverified fleet/backend/runtime path. + +## Checkpoint: qwen25-probe-quality-batch-task-abort + +Status: aborted validation run; useful harness failure after backend-choice improvement. + +Date: 2026-07-07 +Project: `endpoint-agent-reasoning` +Endpoint: `qwen25-probe-quality-2152` +Fleet: `probe-quality-mixed` + +### What Improved + +- Claude chose a task probe instead of service-first. +- The task landed on JarvisLabs `L4-1x` at `$0.44/hr`, despite cheaper RunPod offers. +- The planned checks went beyond host visibility: vLLM/Torch/CUDA import, local server start, + `/health`, `/v1/models`, and a local chat completion request. + +### What Was Still Wrong + +- The probe was encoded as one batch `commands` chain instead of a long-lived task plus attach/SSH. +- No instance volumes were configured. Run JSON showed `configuration.volumes=[]`, + `job_spec.volumes=[]`, and `runtime.volume_names=[]`. +- Claude still used imprecise backend wording: `jarvislabs+runpod (container-style)`. + +### Cleanup + +- Endpoint `qwen25-probe-quality-2152`: `stopped` +- Task `qwen25-probe-quality-2152-1`: `terminated`, `termination_reason=terminated_by_user` +- Temporary fleet `probe-quality-mixed`: deleted + +### Fix Applied After Review + +- Endpoint prompt and `skills/dstack-prototyping/SKILL.md` now require long-lived interactive probes + when attach/SSH is available: keep the probe alive with `sleep infinity` or equivalent, attach/SSH + into it, and run bounded checks inside the live environment. +- Batch task commands are now explicitly reserved for unavailable attach/SSH or truly one-shot + checks. +- Prompt/skill now require optional instance cache mounts for Hugging Face-style model caches when + useful, and require the agent to record why cache mounts were omitted. + +### Verification Commands + +```bash +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +``` + +Observed result: + +- focused agent pytest: `31 passed` +- broader endpoint/preset/pipeline pytest: `117 passed, 50 skipped` diff --git a/endpoint-agent-harness-test-plan.md b/endpoint-agent-harness-test-plan.md index e11e21c613..db2327c337 100644 --- a/endpoint-agent-harness-test-plan.md +++ b/endpoint-agent-harness-test-plan.md @@ -14,8 +14,8 @@ If a test does not answer that question, it is not a useful harness test. ### Agent runtime facts to rely on -- Claude Code print/headless mode supports `--output-format stream-json`, `--json-schema`, `--max-turns`, and `--max-budget-usd`. These are useful for subprocess integration, structured final reports, and API-spend caps. -- `--max-budget-usd` caps Claude API spend only. It does not cap GPU spend from dstack runs. The harness must separately track candidate run lifetimes and stop/abort GPU candidates. +- Claude Code print/headless mode supports `--output-format stream-json`, `--json-schema`, `--max-turns`, and `--max-budget-usd`. These are useful facts for later runtime governance, but v1 does not expose an endpoint agent-budget field. +- `--max-budget-usd` caps Claude API spend only. It does not cap GPU spend from dstack runs. Budget/cost governance must be designed later with durable per-session accounting before it becomes user-facing. - Claude Code plugins/skills are useful for packaging context, but they are not the harness. A plugin can expose skills; it does not give us state, candidate accounting, cleanup, resume, spend tracking, or verification gates. - Public harness engineering writeups emphasize the same lesson: harness design changes outcomes materially, and useful harnesses rely on traces, self-verification/evaluator separation, context handoff artifacts, and controlled execution loops. @@ -157,6 +157,58 @@ Required checks: - final verification request is recorded - endpoint constraints were not violated in previews/submissions - final preset, if saved, includes final service YAML and replica resource evidence +- task/dev probes used for promotion exercised the intended serving stack, not only host/GPU + visibility + +### Placement/Experiment Observer + +For create-recipe e2e runs, the observer must also check that the agent did not +blindly choose the first cheap placement: + +- allowed fleets were used for offer inspection; global offers alone are not enough; +- if a broad fleet exposes multiple backends, the agent compared backend/runtime + characteristics, not just fleet names; +- viable reusable or inspectable placements were identified when present: VM-based, + SSH, Kubernetes, or any backend/runtime path where tasks/dev environments can reuse + image/package/model cache or support interactive diagnosis; +- if such a placement was viable under the endpoint constraints, the first paid + experiment was normally a task or dev-environment style probe, not the final service; +- if the agent chose container-style placement or service-first, `progress.jsonl` contains + a concrete reason, such as no viable reusable offer, + constraint violation, insufficient GPU/disk, much higher price, no useful cache + persistence, or final URL/probe behavior being the only remaining unknown; +- the agent treated hourly price as a constraint, not the objective. A cheaper + container offer is not automatically better than a slightly more expensive reusable + placement if the reusable placement can reduce total iteration time, repeated model + downloads, image churn, or debugging risk; +- the final service was submitted only after the probe removed the main uncertainty, or + after the agent recorded why a probe would not help. +- task-first is not the same as interactive SSH/dev-environment proof. Record whether the + agent had attach/SSH/dev-environment available, whether it used it, and why not if it + skipped it. +- if attach/SSH is available, a probe task should normally stay alive while the agent runs + inspection commands through attach/SSH. A task that packs all checks into one shell + command chain is only acceptable for a genuinely one-shot question or unavailable + attach/SSH. +- a probe is useful only if it reaches the intended recipe evidence: selected image or + install path, Python/framework runtime, model/auth/cache path when feasible, serving + command/port, and ideally a local health or model API request. `nvidia-smi` alone is + host evidence, not service recipe proof. +- for Hugging Face-style model serving, check whether probe and final service YAMLs use + useful optional instance cache mounts, such as Hugging Face and package caches. If not, + the agent must explain why repeated downloads/setup are acceptable for that backend and + model size. +- final service verification remains the success gate. If the final service fails a real + model request, the correct result is another experiment or terminal failure, not a + successful final report based on task/dev evidence. + +This is not a product rule that forbids container backends. It is a harness test: +when reusable/inspectable placement would make the loop faster or more reliable, the +agent should notice and use it. + +Run this as a ladder. First use a cheaper scenario where reusable placement exists +inside a modest budget. Then repeat later with broader/more expensive approved hardware +so the harness is not accidentally tuned only for low-cost small-model cases. ## Required Workspace Artifacts @@ -173,7 +225,6 @@ Required fields: "endpoint_name": "qwen-endpoint-agent-smoke", "model": "Qwen/Qwen3-0.6B", "phase": "research|capacity|experiment|verify|success|failure", - "max_agent_budget": 2.0, "max_hourly_price": 0.3, "started_at": "ISO-8601", "updated_at": "ISO-8601" @@ -196,11 +247,13 @@ One JSON object per source: } ``` -### `hardware_reasoning.md` +### Decision Trail -Purpose: make the hardware decision reviewable. +Purpose: make deployment decisions reviewable without forcing the agent to write +separate markdown notes. -Must include: +Must be visible through `progress.jsonl`, `submissions.jsonl`, `sources.jsonl`, +`verification.json`, and `final_report.json`: - model size and serving mode - expected framework @@ -358,7 +411,6 @@ type: endpoint name: qwen-endpoint-contract model: Qwen/Qwen3-0.6B preset_policy: create -max_agent_budget: 0.25 spot_policy: on-demand max_price: 0.3 ``` @@ -393,7 +445,6 @@ type: endpoint name: qwen-endpoint-no-offers model: Qwen/Qwen3-0.6B preset_policy: create -max_agent_budget: 0.25 spot_policy: on-demand max_price: 0.001 ``` @@ -401,7 +452,7 @@ max_price: 0.001 Pass: - No paid run submitted. -- `sources.jsonl` and `hardware_reasoning.md` explain that model is deployable but constraints block capacity. +- `sources.jsonl`, `progress.jsonl`, and the failure report explain that the model is deployable but constraints block capacity. - Failure summary is concise. Fail: @@ -430,7 +481,6 @@ type: endpoint name: qwen-endpoint-agent-smoke model: Qwen/Qwen3-0.6B preset_policy: create -max_agent_budget: 2.0 spot_policy: on-demand max_price: 0.3 ``` @@ -636,6 +686,11 @@ Development live-test order: 5. If the issue is dstack/backend, reproduce it outside endpoints and report it in `endpoint-agent-backend-troubleshooting.md`. +For the next probe-path e2e, success is not just "the endpoint eventually runs." The agent must +show the intended loop: submit a long-lived task/dev probe, attach/SSH into it, run real recipe +checks inside the live environment, then promote a clean service and verify that service through +the model API. A batch task that embeds the whole investigation in `commands` is a harness failure. + ## Server Integration Gates Only harden endpoint worker behavior after the real server path or unit tests show it is @@ -644,7 +699,7 @@ needed. Avoid speculative scaffolding. ### Gate A: Runtime - Claude Code subprocess can run non-interactively from server env. -- `--max-budget-usd` is honored for API spend. +- Budget/cost governance is deferred until it can be enforced durably per endpoint agent session. - Stream JSON parsing does not block. - Large tool outputs are kept out of endpoint status. - Debug trace captures enough detail. @@ -706,7 +761,7 @@ Prompt wording, skills, plugins, or runtime flags are implementation details. Th - Keep the "one endpoint config to verified service" UX. - Make the generated service YAML inspectable and save it as preset provenance. -- Record the engine/framework choice and important serving flags in `sources.jsonl` / `hardware_reasoning.md`. +- Record the engine/framework choice and important serving flags in `sources.jsonl`, `verification.json`, and the final report. - Record enough final-run metadata to reproduce: model, framework, image/install path, command, resources, observed replica resources, and verification request. - Treat "no hidden black box" as a v1 principle: if the agent made a deployment decision, the workspace artifacts should show why. - Keep first verification functional: a real model request proves the endpoint works. diff --git a/endpoint-e2e-testing-report.md b/endpoint-e2e-testing-report.md index af91e47650..1a6e6fbc7d 100644 --- a/endpoint-e2e-testing-report.md +++ b/endpoint-e2e-testing-report.md @@ -37,7 +37,6 @@ spot_policy: on-demand max_price: 0.5 preset_policy: create -max_agent_budget: 3 ``` Fleet config: @@ -140,7 +139,7 @@ The initial agent behavior also used a candidate run name too close to the endpo Earlier endpoint logs were too noisy in some cases: large CLI output, service logs, and trace-like content could leak into user-facing endpoint logs. The current run is better, but the logging contract still needs hardening. -The endpoint stayed in `clauding` after the service was already healthy because the server intentionally waits for the agent's `final_report.json`. In this run: +The endpoint stayed in `prototyping` after the service was already healthy because the server intentionally waits for the agent's `final_report.json`. In this run: ```text verification.json 10:07:15 @@ -234,7 +233,7 @@ Endpoint user logs need a stricter contract. They should contain major agent mil The first progress line is still too late. The user should see early progress immediately after the agent starts: what it is checking, what constraints it sees, and whether it is researching, submitting, waiting, or verifying. -`clauding` can look stuck when the service is already healthy but the agent is still verifying or writing the final report. We need better progress events, not necessarily another status. +`prototyping` can look stuck when the service is already healthy but the agent is still verifying or writing the final report. We need better progress events, not necessarily another status. The agent still needs much better deployment judgment. The `latest` image failure was caught and recovered from, but the next iterations should make image selection, driver compatibility, model memory sizing, and framework choice more systematic. @@ -246,7 +245,7 @@ The current e2e did not test server restart while Claude is still running. We ha The current e2e did not test multiple server instances with Postgres. The endpoint worker uses the existing lock/pipeline pattern, but the multi-server case still needs an explicit integration test. -The current e2e did not test budget interruption behavior. `max_agent_budget` is passed/configured, but we still need a real test proving accumulated agent spend is tracked per endpoint provisioning attempt and interrupts safely. +The current e2e did not test budget interruption behavior. Agent budget/cost governance is deferred until dstack can persist and enforce spend per endpoint agent session. Debug traces may still contain sensitive command output if the agent runs unsafe inspection commands. This needs a concrete redaction policy before broader testing. @@ -383,6 +382,367 @@ Follow-up UX cleanup: stopped endpoints now hide the linked service run in `dsta the database for lifecycle/history purposes, but the user-facing field represents the current live backing service, not old run history. +## 2026-07-07 Same-Host Restart And Resource Envelope E2E + +Goal: validate the latest prompt/session changes on a fresh create-policy RunPod endpoint, including +same-host server restart while Claude is running, final service resource envelope quality, final +report handoff, preset save, and stop cleanup. + +Endpoint config: + +```yaml +type: endpoint +name: qwen2-05b-restart-1110 +model: Qwen/Qwen2-0.5B-Instruct + +backends: + - runpod +fleets: + - endpoint-e2e-runpod +spot_policy: on-demand +max_price: 0.5 + +preset_policy: create +``` + +Workspace: + +```text +~/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace +``` + +Server restart result: + +```text +endpoint id: 6a9ce2c9-2310-4210-9533-0acf47e49d27 +Claude pid before restart: 83844 +Claude pid after restart: 83844 +duplicate Claude process: no +endpoint state: prototyping +session/workspace: reused +``` + +This validates the same-host case: restarting the server while Claude was already running did not +create a second agent process or a second agent session. + +Final service run: + +```text +run name: qwen2-05b-restart-1110-1 +run id: c1be5e1a-6c49-4ef5-926e-4d2171a03d98 +backend: runpod +region: EU-CZ-1 +gpu: NVIDIA GeForce RTX 3090, 24GB +price: $0.46/hr +image: vllm/vllm-openai:v0.24.0 +model: Qwen/Qwen2-0.5B-Instruct +``` + +The final service YAML used the intended scheduling envelope instead of copying exact hardware: + +```yaml +resources: + gpu: 16GB..24GB:1 + disk: 30GB.. +``` + +The saved preset preserved that distinction: + +```text +preset: qwen-qwen2-0-5b-instruct-c1be5e1a +scheduling: cpu=2.. mem=8GB.. disk=30GB.. gpu=16GB..24GB:1 +validation: cpu=32 mem=125GB disk=100GB gpu=RTX3090:24GB:1 +``` + +This is the important preset-quality result from the run: reuse planning can stay broad enough to +find equivalent 16-24GB offers, while exact verified placement remains inspectable through +`dstack endpoint preset get --json`. + +Verification and handoff: + +```text +service running: 09:22 UTC +model API HTTP 200: 09:22:46 UTC +final_report.json: 09:23:24 UTC +endpoint running: 09:23:52 UTC +endpoint URL: /proxy/services/endpoint-e2e/qwen2-05b-restart-1110-1/v1 +``` + +The endpoint correctly stayed `prototyping` after the service was healthy until the agent produced a +structured verification report. Once Claude exited, the worker consumed the report, linked the +service run, saved the preset, and moved the endpoint to `running`. + +Stop cleanup: + +```text +endpoint stop requested: 09:24 UTC +service run status: terminated +termination reason: stopped_by_user +endpoint status: stopped +final run cost: $0.0484 +Claude reported cost: $1.1923 +``` + +The stop path terminated the RunPod service and left the endpoint visible as `stopped`. + +Remaining issues from this trace: + +- Claude first wrote `backend: [runpod]` instead of `backends: [runpod]`, and initially omitted + `fleets`; the workspace CLI guard caught both before any paid submission and Claude corrected the + YAML. The guard worked, but the prompt/skill should still reduce this waste. +- The endpoint log stream still included Claude assistant stdout such as "Let me..." and the final + markdown summary, even though `progress.jsonl` itself was clean. Fixed after the run: endpoint logs + now come from `progress.jsonl` and explicit server lifecycle messages only; Claude stdout remains + in trace/artifacts. +- The agent treated early proxy `404 Service not found` as a retryable registration/startup delay and + then verified successfully. That behavior was good, but the prompt should keep this tied to run JSON + and service readiness evidence, not blind retries. +- The report recorded that CUDA/FlashAttention/NCCL initialized successfully, but still did not + capture a direct `nvidia-smi` host driver string. For presets, this should remain explicit: + runtime worked; exact host driver was not recorded. +- Another preset for the same model already existed from earlier testing. Duplicate learned recipes + are acceptable for v1 inspection; v1 selection is intentionally simple and deterministic. + +Reuse preview after cleanup: + +```text +configuration: qwen2-05b-reuse-1110.dstack.yml +preset_policy: reuse +selected preset: qwen-qwen2-0-5b-instruct-532ddf05 +offers: 7 matching RunPod offers, first RTX 2000 Ada at $0.24/hr +submitted: no +``` + +This proved that a Qwen2 preset is reusable without Claude, but it did **not** prove reuse of the +new `c1be5e1a` preset. The planner selected the older duplicate recipe, whose scheduling resources +are looser (`gpu: 1`, `disk: 100GB`) than the new resource-envelope recipe. That is now accepted v1 +behavior: try stored recipes in order and use the first recipe whose normal run plan has available +offers. Automatic ranking/update remains deferred. + +## 2026-07-07 Qwen2.5 Preset Reuse And CLI UX Check + +Goal: validate the preset-reuse path for the model-level recipe format after the latest CLI/storage +changes, and inspect whether the endpoint/preset tables are understandable during normal use. + +Test directory: + +```text +~/dstack-endpoints-demo/e2e-2026-07-07 +``` + +Important testing note: running `uv run dstack` from outside the repo can pick up the installed +package version instead of this branch. Use the branch binary for isolated e2e runs: + +```text +/Users/dstack/dstack/.venv/bin/dstack ... +``` + +Endpoint config: + +```yaml +type: endpoint +name: qwen25-05b-reuse-check +model: Qwen/Qwen2.5-0.5B-Instruct + +preset_policy: reuse +fleets: + - endpoint-e2e-runpod +backends: + - runpod +spot_policy: on-demand +max_price: 0.5 +``` + +Plan/preview: + +```text +preset: Qwen/Qwen2.5-0.5B-Instruct +recipe: 4a73893b +offers: 5 matching RunPod offers under the endpoint constraints +``` + +Paid run: + +```text +endpoint submitted: 2026-07-07 10:33:46 UTC +service run: qwen25-05b-reuse-check-serving +actual placement: runpod CA-MTL-1, NVIDIA RTX A5000 24GB +price: $0.27/hr +``` + +The service emitted vLLM startup logs, downloaded and loaded the model, compiled/warmed up, exposed +the OpenAI-compatible API, and served real chat-completion requests. The endpoint stayed +`provisioning` until the existing service probe/registration path passed, then moved to `running`. + +Direct endpoint verification: + +```text +POST /proxy/services/endpoint-e2e/qwen25-05b-reuse-check-serving/v1/chat/completions +status: 200 +model: Qwen/Qwen2.5-0.5B-Instruct +reply: non-empty assistant message +``` + +Stop result: + +```text +endpoint status: stopped +service run: terminated/stopped +active spend: no live GPU run left +``` + +What this proves: + +- The no-Claude preset-reuse path still works after moving presets to model-level recipes. +- The server waits for real service readiness, not only process startup. +- A learned recipe can plan on one currently available offer and land on another valid offer as + provider availability changes; the actual placement belongs in run/provisioning history and + future validation evidence, not in the apply table as a promise. +- Empty endpoint logs for `preset_policy: reuse` are acceptable for now because no agent is running. + If users need richer preset-path progress later, it should come from server lifecycle events, not + service stdout. + +CLI/storage fixes made after this check: + +- `dstack endpoint` now hides the backing service run in the default table and shows `POLICY`. + The service run remains visible in verbose/detail/JSON output as debugging information. +- Default endpoint listing now behaves like `ps`: show unfinished endpoints; if none exist, show the + latest finished endpoint. +- `dstack endpoint preset` now groups rows by model and shows only the scheduling GPU by default. + `-v` shows full service scheduling resources. Child `recipe=0`, `recipe=1`, ... rows appear + only when a model has multiple recipes. Validation counts were removed from the compact table; + exact validation evidence remains in `dstack endpoint preset get --json`. +- The agent harness now asks Claude to make final GPU service resources vendor-aware when the + selected/proven hardware is vendor-specific, e.g. `gpu: nvidia:16GB..24GB:1`. The preset loader + preserves existing vendorless local recipes instead of rewriting them on read. +- The local preset store now merges duplicate files by model for list/get/planning, and save/delete + collapse or remove all files for that model. In the live project, + `dstack endpoint preset get 'Qwen/Qwen2-0.5B-Instruct' --json` now returns both recipe ids + (`c04afca5`, `79a6b0b7`) consistently with the list output. + +Verification after code changes: + +```text +endpoint-focused pytest: 185 passed, 72 skipped +focused preset/CLI pytest: 57 passed +ruff: All checks passed +pyright: 0 errors +live endpoint table: default shows NAME/MODEL/STATUS/POLICY/CREATED +live preset table: default shows MODEL/GPU, grouped by model with recipe ordinals only when needed +``` + +## 2026-07-07 Broad-Fleet Vendor-Aware Harness Check + +Goal: remove the RunPod-only test bias, verify that the endpoint does not force a backend, and prove +that new learned recipes can be vendor-aware without mutating older local vendorless recipes. + +Test fleet: + +```yaml +type: fleet +name: endpoint-e2e-gpu +nodes: 0..1 +resources: + gpu: 1 + disk: 30GB.. +backends: + - lambda + - verda + - jarvislabs + - runpod +spot_policy: auto +max_price: 0.5 +idle_duration: 5m +``` + +Important observation: the fleet was no longer RunPod-only, but the currently available matching +offers under `$0.5/hr` were still all RunPod. Lambda, Verda, JarvisLabs, and Vast.ai returned no +matching GPU offers in this local project even with relaxed price checks. So the agent selecting +RunPod in this test was explained by current capacity/offer reality, not by endpoint config. + +Endpoint config: + +```yaml +type: endpoint +name: qwen25-broad-fleet +model: Qwen/Qwen2.5-0.5B-Instruct +preset_policy: create +fleets: + - endpoint-e2e-gpu +spot_policy: auto +max_price: 0.5 +``` + +Result: + +```text +service run: qwen25-broad-fleet-1 +run id: b601b053-0b65-4b88-a1ba-53a4ecd15423 +actual placement: runpod EU-RO-1, NVIDIA RTX 2000 Ada 16GB +price: $0.24/hr +endpoint state: running, then stopped +``` + +The agent wrote a clean final service YAML: + +```yaml +fleets: + - endpoint-e2e-gpu +max_price: 0.5 +spot_policy: auto +resources: + gpu: nvidia:16GB..24GB:1 + disk: 30GB.. +``` + +The agent verified the model with a real OpenAI-compatible chat request. The server consumed +`final_report.json`, linked the endpoint to the final service run, saved a new vendor-aware recipe, +and marked the endpoint `running`. + +Reuse preview after stopping the live endpoint: + +```text +configuration: qwen25-broad-reuse-preview.dstack.yml +preset_policy: reuse +selected model preset: Qwen/Qwen2.5-0.5B-Instruct +selected recipe: 4a73893b +offers: 10 matching RunPod offers, first RTX 2000 Ada at $0.24/hr +submitted: no +``` + +The preview initially showed no offers while the only `nodes: 0..1` fleet slot was occupied by the +live endpoint. After stopping the endpoint, the same preview showed offers. That behavior is +conventional capacity accounting, not a preset matching bug. + +Issues found: + +- Endpoint logs were blank for the first ~minute while Claude inspected the workspace/fleet. Normal + endpoint logs need earlier useful progress, even though debug traces already captured commands. +- Older local same-model recipes remain selectable before newer vendor-aware recipes if they are + provisionable. That is now the explicit v1 rule: try stored recipes in order and use the first + recipe whose normal run plan has available offers. Explicit `recipe: ` selection is deferred. +- The agent still wrote a slightly stale final image (`vllm/vllm-openai:v0.6.6`) for a tiny model. + It worked, but image choice needs stronger grounding against current vLLM/SGLang guidance in + harder scenarios. + +Follow-up log-smoke fix: + +```text +endpoint: qwen25-log-start-smoke +purpose: create-policy smoke, stopped before any service submission +result: endpoint logs showed the server startup progress line and Claude's first progress line +service: no new service run submitted +``` + +Normal endpoint logs now get an immediate server-written line when the detached Claude session starts: + +```text +Starting endpoint prototyping agent for Qwen/Qwen2.5-0.5B-Instruct. Allowed fleets: endpoint-e2e-gpu. The agent will inspect offers, choose a service recipe, deploy it, and verify the model API before the endpoint becomes running. +``` + +The prompt now also requires Claude's first workspace action to be a short `progress.jsonl` message +before inspecting state, fleets, offers, recipes, logs, or model docs. + ## Assessment This was the first meaningful proof that the endpoint idea can work end to end: endpoint config in, real agent work, real service deployed, real model request verified, endpoint running, preset saved. @@ -393,12 +753,859 @@ The biggest remaining risk is still the harness quality. For v1, the server plum ## Next Tests -1. Run another small no-preset agent deployment and compare behavior against the first Qwen run. -2. Run a server restart test while the endpoint is still `clauding`. -3. Test endpoint stop while Claude is running and after a candidate run has been submitted. -4. Test budget interruption once spend is persisted per endpoint provisioning attempt. -5. Run a private/Hugging Face gated model test with environment handling. -6. Run a larger common model after confirming budget and hardware. -7. Test one dev-environment prototyping flow where the agent experiments before submitting the final service. -8. Add log/trace redaction tests. -9. Add regression tests for run-name ambiguity, final-report-to-running delay, and preset reuse. +1. Reduce the YAML/property waste exposed by the guard (`backend` vs `backends`, omitted `fleets`) + without weakening the guard. +2. Run existing-fleet scenarios that expose both reusable/inspectable backend placement + (VM-based, SSH, or Kubernetes) and container-style placement if possible. Start with a cheaper + scenario, then test broader/more expensive hardware separately after budget approval. Each test + should verify that the agent compares backend/runtime characteristics inside the allowed fleets, + treats hourly price as a constraint rather than the objective, prefers the reusable placement when + viable, starts with a task/dev-style probe when that removes meaningful uncertainty, and records a + concrete reason if it chooses container placement or service-first. +3. Test private/Hugging Face gated model env handling. +4. Run a larger common model after confirming budget and target hardware. +5. Test multiple server instances with Postgres; same-host restart is now tested, multi-host is not. +6. Add log/trace redaction tests and keep endpoint logs limited to progress-level messages. +7. Add regression tests for final-report-to-running handoff, preset reuse, and cleanup of non-final + submitted runs after restart. + +## 2026-07-07 no-cost recipe-selection check + +Regression tests now cover the v1 selection rule directly: + +- if the first stored recipe has no available offers and the next recipe does, the second recipe is + selected and the first is retained as the first unprovisionable match; +- if the first stored recipe has available offers, planning stops there and does not inspect later + recipes. + +Live no-cost preview from `~/dstack-endpoints-demo/e2e-2026-07-07`: + +```text +configuration: qwen25-broad-reuse-preview.dstack.yml +project: endpoint-e2e +preset_policy: reuse +selected model preset: Qwen/Qwen2.5-0.5B-Instruct +selected recipe: 4a73893b +offers: 10 matching offers, first RunPod RTX 2000 Ada at $0.24/hr +submitted: no +``` + +The preview was run with `n` at the confirmation prompt, so it did not create an endpoint and did +not invoke Claude. Running the same preview by absolute path from the repo still hits the existing +generic CLI `relative_to(Path.cwd())` path error for configs outside the current directory; that was +observed but intentionally left untouched here. + +## 2026-07-07 prompt/guard hardening for YAML constraints + +Live traces repeatedly showed Claude confusing service YAML fields with CLI flags, especially +writing singular `backend` or omitting `fleets` and relying on the guard to catch it. The guard +stays as the hard safety net, but the prompt/skill should reduce those failed previews. + +Changes made: + +- endpoint prompt now explicitly separates plural service YAML fields (`fleets`, `backends`, + `regions`, `instance_types`) from singular CLI flags (`--fleet`, `--backend`, `--region`, + `--instance-type`); +- `dstack-prototyping` now repeats that distinction near the fleet-filtered offer guidance and the + final service promotion checklist; +- workspace CLI guard now returns targeted errors for singular run-config keys such as `fleet` and + `backend`, before falling through to generic missing-constraint messages. + +No paid run was needed for this step. Tests covered prompt inclusion and guard behavior. + +## 2026-07-07 no-cost capacity preflight + +Current project/fleet state: + +```text +project: endpoint-e2e +fleet: endpoint-e2e-gpu +shape: nodes 0..1, gpu:1, disk 30GB.., spot auto, max_price $0.5 +backends: lambda, verda, jarvislabs, runpod +live instances: none +active endpoints: none +``` + +The last visible run row was only history: + +```text +qwen25-broad-fleet-1: terminated, stopped_by_user, price $0.24/hr +``` + +Offer checks: + +- `endpoint-e2e-gpu` under `$0.5/hr`: RunPod only. +- `endpoint-e2e-gpu` under `$1/hr`: still RunPod only. +- Lambda, Verda, and JarvisLabs through this fleet: no matching offers. +- Nebius is not configured in `~/.dstack/server/config.yml`. + +Conclusion: the current project cannot test the desired VM/SSH-capable prototyping path. A paid +agent e2e using the current fleet will most likely be a RunPod/container-backed test again. That is +still useful for checking the prompt/guard fix, but it will not validate dev-environment-style +reuse on a VM fleet. + +## 2026-07-07 backend-placement e2e setup + +Goal: create a no-spend setup that can actually test whether the agent prefers reusable/inspectable +backend placement and task-first probing when that makes sense. The important distinction is that +fleets define allowed capacity, while backend placement determines the prototyping strategy. + +Test directory: + +```text +~/dstack-endpoints-demo/backend-placement-2026-07-07 +``` + +Project: + +```text +endpoint-agent-reasoning +``` + +Fleet template applied with `nodes: 0..1`, so no GPU spend was started: + +```yaml +type: fleet +name: reusable-vs-container + +nodes: 0..1 + +resources: + gpu: nvidia:24GB:1 + disk: 100GB.. + +backends: + - jarvislabs + - runpod + +spot_policy: auto +max_price: 1.5 +idle_duration: 30m +``` + +Fleet-filtered offers under `$1.5/hr`: + +```text +runpod RTX A5000 24GB $0.27/hr +runpod RTX A5000 24GB $0.27/hr +runpod L4 24GB $0.39/hr +jarvislabs L4 24GB $0.44/hr +runpod RTX 3090 24GB $0.46/hr +runpod RTX PRO 4000 $0.57/hr +``` + +This is a useful harness test shape: cheaper container-style placements exist, but there is also a +viable JarvisLabs L4 placement inside the same allowed fleet. The next paid endpoint run should +validate whether the agent compares backend/runtime characteristics, treats hourly price as a +constraint rather than the objective, prefers the reusable/inspectable placement when it improves the +total experiment loop, starts with a task/dev-style probe before the final service, and records a +concrete reason if it chooses RunPod or service-first. + +Prepared endpoint config: + +```yaml +type: endpoint +name: qwen25-7b-placement-choice +model: Qwen/Qwen2.5-7B-Instruct + +preset_policy: create +fleets: + - reusable-vs-container +spot_policy: auto +max_price: 1.5 +``` + +No-spend endpoint preview showed the create-policy agent path and exited at confirmation. The paid +run result is below. + +## 2026-07-07 backend-placement / task-first e2e + +Endpoint: + +```text +project: endpoint-agent-reasoning +endpoint: qwen25-7b-placement-choice +model: Qwen/Qwen2.5-7B-Instruct +fleet: reusable-vs-container +``` + +Allowed capacity intentionally exposed both cheaper container placement and a reusable/inspectable +VM-style placement: + +```text +runpod RTX A5000 24GB $0.27/hr +runpod L4 24GB $0.39/hr +jarvislabs L4 24GB $0.44/hr +runpod RTX 3090 24GB $0.46/hr +``` + +Observed agent route: + +- Chose JarvisLabs L4 over cheaper RunPod offers because JarvisLabs was reusable/inspectable and + could keep a warm instance inside the same allowed fleet. +- Submitted a task first: `qwen25-7b-placement-choice-1`. +- Then submitted the final service: `qwen25-7b-placement-choice-2`. +- The service reused the warm JarvisLabs L4 instance and reached `running`. +- Final verification succeeded through the dstack service URL: `/v1/models=200` and + `/v1/chat/completions=200` for `Qwen/Qwen2.5-7B-Instruct`. +- The endpoint moved to `running`, saved the learned recipe, then was stopped cleanly. +- The temporary fleet was deleted; a later fleet listing for `endpoint-agent-reasoning` showed no + active fleets. + +Final service evidence: + +```text +run id: dbc160ff-8e44-4beb-bdcb-7a294c1a5d44 +backend: jarvislabs +region: india-noida-01 +instance: L4-1x +gpu: NVIDIA L4 24GB +driver: 580.126.20 +CUDA reported: 13.0 +price: $0.44/hr +image: vllm/vllm-openai:v0.11.0 +``` + +What this proves: + +- The agent can notice a reusable/inspectable backend when a cheaper container backend is also + available inside the same allowed fleet. +- The agent can choose task-first before the final service. +- A warm reusable instance can reduce the final service placement loop. +- The server correctly waited for the agent's final service verification report before marking the + endpoint `running`. + +What this does not prove: + +- It did not use SSH or a dev environment. This proves task-first on a reusable backend, not an + interactive SSH/dev loop. +- The task probe was too shallow. It successfully observed `nvidia-smi`, host driver, and GPU + memory, but then failed on `/bin/sh: 1: python: not found`. The agent over-interpreted that as + full image/runtime compatibility. The final service still proved the endpoint, but the probe did + not prove framework import, `torch.cuda`, model download, local server start, or local API shape. + +Required harness correction from this run: + +- A task/dev probe must exercise the intended serving image/runtime/command, not only the host. +- `nvidia-smi` is host evidence, not service recipe evidence. +- If a probe exits before framework/runtime/server checks, it is failed or inconclusive evidence. +- The clean service remains the final authority. If final service verification fails, the agent must + go back to task/dev or submit a changed service; it must not write success. + +## 2026-07-07 probe-quality rerun aborted after service-first decision + +Goal: validate the probe-quality prompt fix with a fresh create-policy endpoint. + +Project and setup: + +```text +project: endpoint-agent-reasoning +endpoint: qwen25-probe-quality-2136 +model: Qwen/Qwen2.5-0.5B-Instruct +fleet: probe-quality-mixed +``` + +The temporary fleet intentionally exposed the same placement tension as the previous validation: + +```text +runpod A5000 24GB $0.27/hr +runpod L4 24GB $0.39/hr +jarvislabs L4 24GB $0.44/hr +runpod RTX3090 $0.46/hr +``` + +What happened: + +- Claude saw JarvisLabs and RunPod in the fleet-filtered offers. +- Claude wrote that both backends were "container-only (no reusable VM/SSH state)". +- Based on that claim, it skipped the task/dev probe. +- It selected the cheaper RunPod A5000 and submitted service `qwen25-probe-quality-2136-1`. +- We stopped the endpoint before waiting for service verification because the run no longer tested + the intended probe-quality fix. + +Cleanup: + +```text +endpoint: qwen25-probe-quality-2136 stopped +service: qwen25-probe-quality-2136-1 terminated_by_user +backend: runpod CA-MTL-1 A5000, $0.27/hr +fleet: probe-quality-mixed deleted +``` + +Why this matters: + +- The issue was not that Claude failed to run a deep enough task probe; it did not run a probe at + all. +- The deeper issue is that the harness let Claude infer backend capability from an offer table and + turn uncertainty into a false statement: "both backends are container-only". +- Once it made that false inference, it optimized back to the cheaper container-style service path. + +Fix made after this aborted run: + +- Endpoint prompt and `dstack-prototyping` now say not to infer "container-only" or "no reusable + state" from offers alone. +- Fleet state matters: `nodes: 0..N`, `idle_duration`, or an idle/running instance may keep capacity + warm for a later task or service. +- If backend reuse/SSH/cache behavior is uncertain, the agent must record uncertainty and choose an + experiment that can resolve it; it must not treat uncertainty as proof that a task would preserve + nothing. +- For create-recipe endpoints, "the model is small" or "the framework is common" is not enough to + skip a task/dev probe on an unverified fleet/backend/runtime path. + +## 2026-07-07 probe-quality rerun aborted after batch-style task probe + +Goal: rerun the same mixed RunPod/JarvisLabs scenario after the backend-capability prompt fix and +watch whether the agent chooses a reusable/inspectable placement and a meaningful probe. + +Project and setup: + +```text +project: endpoint-agent-reasoning +endpoint: qwen25-probe-quality-2152 +model: Qwen/Qwen2.5-0.5B-Instruct +fleet: probe-quality-mixed +``` + +Fleet-filtered offers again included cheaper RunPod A5000/L4 options and JarvisLabs L4: + +```text +runpod A5000 24GB $0.27/hr +runpod L4 24GB $0.39/hr +jarvislabs L4 24GB $0.44/hr +runpod RTX3090 $0.46/hr +``` + +What improved: + +- Claude did not default back to the cheapest RunPod service. +- Claude submitted a task first: `qwen25-probe-quality-2152-1`. +- The task landed on JarvisLabs `L4-1x` at `$0.44/hr`. +- The intended checks were deeper than `nvidia-smi`: vLLM/Torch/CUDA import, local vLLM server + start, `/health`, `/v1/models`, and `/v1/chat/completions`. + +What was still wrong: + +- The task was a batch script, not an interactive probe. The full investigation was encoded into + one shell command chain instead of starting a long-lived task and attaching/SSH-ing into it. +- The task had no instance cache mounts. Run JSON confirmed: + +```text +configuration.volumes=[] +job_spec.volumes=[] +runtime.volume_names=[] +``` + +- The agent still wrote `jarvislabs+runpod (container-style)` too loosely. It did not use that claim + to skip the probe this time, but backend capability still needs stronger evidence than names in an + offer table. + +Cleanup: + +```text +endpoint: qwen25-probe-quality-2152 stopped +task: qwen25-probe-quality-2152-1 terminated_by_user +backend: jarvislabs L4-1x, $0.44/hr +fleet: probe-quality-mixed deleted +``` + +Fix made after this run: + +- Endpoint prompt and `dstack-prototyping` now say that when SSH/attach is available, a task probe + should be long-lived (`sleep infinity` or equivalent) and inspected through attach/SSH; batch task + commands are only for unavailable attach/SSH or truly one-shot checks. +- Prompt/skill now require optional instance cache mounts for Hugging Face-style model caches when + useful, e.g. `/dstack-cache/huggingface` to `/root/.cache/huggingface` with `optional: true`, and + they require the agent to record why cache mounts were omitted. + +Checks after the fix: + +```text +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +``` + +Observed result: + +```text +31 passed +117 passed, 50 skipped +``` + +## 2026-07-07 probe-quality rerun aborted after cached batch task + +Goal: rerun after the long-lived-interactive-probe and cache-mount prompt update. + +Project and setup: + +```text +project: endpoint-agent-reasoning +endpoint: qwen25-probe-quality-2202 +model: Qwen/Qwen2.5-0.5B-Instruct +fleet: probe-quality-mixed +``` + +What improved: + +- The generated probe task included an optional Hugging Face instance cache mount: + +```yaml +volumes: + - instance_path: /dstack-cache/huggingface + path: /root/.cache/huggingface + optional: true +``` + +- The task again planned recipe-level checks beyond `nvidia-smi`: vLLM import, local vLLM server + start, health check, and local chat completion. + +What was still wrong: + +- The task was still a batch `commands` chain, not a long-lived task plus attach/SSH inspection. +- Claude again described `jarvislabs+runpod` as `container-style` based on fleet/offers instead of + grounded backend capability. +- It selected/submitted RunPod L4 instead of the slightly more expensive JarvisLabs path, so the + backend-placement behavior regressed from the previous run. +- The run was stopped immediately because it no longer tested the intended interactive probe path. + +Cleanup: + +```text +endpoint: qwen25-probe-quality-2202 stopped +task: qwen25-probe-quality-2202-1 terminated_by_user +backend: runpod NVIDIA L4 +fleet: probe-quality-mixed deleted +``` + +Conclusion: + +Prompt/skill prose is no longer enough for this part of the harness. The next fix should be +structural: pass explicit backend/fleet capability context from the server into the workspace and +consider a pre-submit guard or required pre-submit artifact when the prompt expects an interactive +probe. More wording alone is unlikely to make Claude reliably choose attach/SSH over a batch task. + +## 2026-07-07 existing functionality e2e + +Goal: before moving to the next endpoint-plan item, re-check the already implemented surfaces +without spending GPU time. + +Project state: + +```text +project: endpoint-e2e +fleet: endpoint-e2e-gpu active, nodes 0..1, no live instances +unfinished endpoints: none +latest visible run row: qwen25-broad-fleet-1, terminated/stopped_by_user +``` + +Read-only checks: + +- `dstack endpoint --project endpoint-e2e -a -n 10` showed stopped endpoint history and no + unfinished endpoint. +- `dstack endpoint get qwen25-broad-fleet --json` returned `stopped`, no linked run/url/error. +- `dstack endpoint logs qwen25-broad-fleet --since 24h` showed endpoint progress lines only, not + backing service logs. +- `dstack endpoint logs qwen25-log-start-smoke --since 24h` showed the server startup line plus + Claude's first progress line. +- top-level `dstack logs qwen25-broad-fleet` and `dstack stop qwen25-broad-fleet -y` looked for a + run and returned `Run qwen25-broad-fleet not found`, so endpoint/run command separation holds. +- `dstack endpoint stop qwen25-broad-fleet -y` returned `Endpoint qwen25-broad-fleet is already + stopped`. + +Preset checks: + +- compact `dstack endpoint preset list` grouped recipes by model and showed GPU only. +- verbose `dstack endpoint preset list -v` showed full scheduling resources. +- `dstack endpoint preset get Qwen/Qwen2.5-0.5B-Instruct --json` returned both recipes with + `validations`. +- `dstack endpoint preset get ...` without `--json` correctly refused with `Use --json to output + the endpoint preset.` + +Apply previews, all answered `n`: + +- `qwen25-broad-reuse-preview.dstack.yml` selected preset model + `Qwen/Qwen2.5-0.5B-Instruct`, recipe `4a73893b`, and showed 10 matching RunPod offers. +- `qwen25-broad-fleet-harness.dstack.yml` showed the create/agent path and confirmation without + submitting. +- `no-fleet-create-preview.dstack.yml` against project `endpoint-agent-choice` showed `The project + has no fleets. Create one before submitting an endpoint.` +- old `qwen25-05b-create.dstack.yml` correctly showed `No fleets match the endpoint configuration` + because it still references deleted fleet `endpoint-e2e-runpod`. + +Issue found and fixed: + +- `dstack endpoint preset list -v` worked, but `dstack endpoint preset -v list` printed compact + output even though the help shape allowed `-v` before the action. Root cause: the `list` + subparser's default `verbose=False` overwrote the parent parser's `verbose=True`. Fixed by making + the child default suppressed; both forms now print verbose output. + +Checks run: + +```text +uv run pytest src/tests/_internal/cli/commands/test_endpoint.py -q +uv run pytest src/tests/_internal/cli/commands/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/services/configurators/test_endpoint.py -q +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py::TestFindMatchingPreset src/tests/_internal/server/services/endpoints/test_claude_agent.py::TestClaudeAgentService::test_agent_cli_guard_explains_singular_run_config_fields -q +uv run ruff check src/dstack/_internal/cli/commands/endpoint.py src/tests/_internal/cli/commands/test_endpoint.py +``` + +Result: existing no-spend endpoint functionality looks coherent after the CLI verbose fix. The main +remaining plan work is not basic CLI behavior; it is still the agent/harness durability and richer +real-run scenarios. + +## 2026-07-07 main loop validation + +Goal: stop polishing peripheral CLI behavior and validate the central loop again: + +```text +endpoint config -> Claude deploys and verifies service -> endpoint running -> preset saved -> reuse preview selects the learned recipe +``` + +Config was kept outside the repo: + +```text +~/dstack-endpoints-demo/e2e-2026-07-07/qwen25-mainloop-1642.dstack.yml +``` + +Endpoint: + +```yaml +type: endpoint +name: qwen25-mainloop-1642 +model: Qwen/Qwen2.5-0.5B-Instruct +preset_policy: create +fleets: + - endpoint-e2e-gpu +spot_policy: auto +max_price: 0.5 +``` + +What happened: + +- Claude inspected the allowed fleet and current offers, then submitted + `qwen25-mainloop-1642-1` with `uv pip install vllm`. +- That first service landed on RunPod RTX 2000 Ada at `$0.24/hr`, reached `running`, then failed at + CUDA initialization. vLLM `0.24.0` pulled a CUDA 13 torch build, while the host driver exposed + CUDA 12.8 (`found version 12080`). +- Claude diagnosed the failure from service logs, recorded it in endpoint progress, and submitted + `qwen25-mainloop-1642-2` using + `vllm/vllm-openai:v0.24.0-cu129-ubuntu2404`. +- The second service reached `running`; `/v1/models` and a real `/v1/chat/completions` request both + returned HTTP 200 with model `Qwen/Qwen2.5-0.5B-Instruct`. +- The server consumed `final_report.json`, linked endpoint `service_run_id`, set the endpoint to + `running`, saved the preset, and `dstack endpoint stop` terminated the backing run. + +Important harness observations: + +- This was a useful real failure: unpinned package installs are risky on container backends because + host driver/CUDA compatibility is not visible in offers. +- Endpoint progress was substantially better than earlier traces: it showed the failed hypothesis, + the driver mismatch, the second attempt, and the final verification. +- The official pinned vLLM image avoided the CUDA mismatch, but it also made the provisioning/startup + phase slower. The agent handled this with bounded polls, though it could still write a progress + line during long image pulls. +- The final service YAML was vendor-aware (`gpu: nvidia:16GB..24GB:1`) and honored the endpoint + fleet/price/spot constraints. + +Preset reuse issue found and fixed: + +- Before the fix, a no-spend `preset_policy: reuse` preview for the same model selected older + recipe `4a73893b`, not the freshly verified recipe `01141556`. +- Root cause: `save_preset` appended new recipes after older recipes, while v1 planning intentionally + uses the first provisionable recipe. +- Fix: saving an incoming learned recipe now moves that recipe to the front of the model preset. + The planner remains simple: first stored recipe with offers wins. +- After re-saving the successful recipe through the preset service, the same no-spend reuse preview + selected recipe `01141556` and showed matching RunPod offers without invoking Claude. + +Checks run: + +```text +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py -q +``` + +Result: the main loop works for this RunPod/Qwen/vLLM case, including recovery from one real runtime +failure and immediate no-agent preset reuse at the apply-plan level. Next main-path work should focus +on learned recipe quality and more representative scenarios, not more CLI surface polish. + +Paid reuse follow-up: + +- Submitted `qwen25-reuse-after-mainloop` with `preset_policy: reuse`. +- Apply selected recipe `01141556`, the freshly verified CUDA-12 vLLM image recipe. +- The service run landed on RunPod CA-MTL-1, NVIDIA RTX A5000 24GB, `$0.27/hr`. +- Provisioning looked stuck for several minutes from dstack alone: endpoint/run stayed + `provisioning` and normal workload logs were initially empty. +- RunPod native REST/GraphQL showed useful intermediate state: the pod was rented, desired status was + `RUNNING`, runtime/ports existed, image was `vllm/vllm-openai:v0.24.0-cu129-ubuntu2404`, and the + machine was RTX A5000. The native API did not expose image-pull percentage through the fields used + here. The HAPI logs URL returned 401/403 with the configured backend API key, so it is not a stable + backend-diagnostic path for dstack. +- The endpoint then reached `running`, service logs showed vLLM startup and repeated HTTP 200 model + requests, and `dstack endpoint stop` terminated the backing run cleanly. + +Diagnostic lesson: when dstack is stuck at `provisioning` with no workload logs, official native +backend APIs are worth checking for pod/runtime/container state. Do not depend on web-app session +tokens or Clerk endpoints; use the backend's configured API credential and print only sanitized +operational fields. + +## 2026-07-07 7B main-loop validation + +Goal: validate the central learning loop on a less trivial public model: + +```text +endpoint config -> Claude chooses hardware and deploys -> service fails for real reasons -> +Claude adjusts -> service answers model API -> endpoint running -> preset saved -> reuse preview +selects the preset without Claude +``` + +Config was kept outside the repo: + +```text +~/dstack-endpoints-demo/e2e-2026-07-07/qwen25-7b-mainloop-1722.dstack.yml +``` + +Endpoint: + +```yaml +type: endpoint +name: qwen25-7b-mainloop-1722 +model: Qwen/Qwen2.5-7B-Instruct +preset_policy: create +fleets: + - endpoint-e2e-gpu +spot_policy: auto +max_price: 0.5 +``` + +What happened: + +- Claude chose a NVIDIA 24GB..48GB envelope for the 7B model, avoiding the earlier too-loose + `gpu: 1` pattern. +- Run `qwen25-7b-mainloop-1722-1` failed before capacity with `failed_to_start_due_to_no_capacity`. +- Run `qwen25-7b-mainloop-1722-2` landed on RunPod A40 and then failed because FlashInfer tried + runtime JIT through `/usr/local/cuda/bin/nvcc`, which was not present. +- Run `qwen25-7b-mainloop-1722-3` switched to `vllm/vllm-openai:v0.24.0`, landed on RunPod + CA-MTL-1 NVIDIA RTX A5000 at `$0.27/hr`, and served the model successfully. +- Final verification used the model endpoint, not service status alone: `/v1/models` returned + `Qwen/Qwen2.5-7B-Instruct` with `max_model_len: 32768`, and `/v1/chat/completions` returned + HTTP 200 with content `dstack verification OK`. +- The endpoint moved to `running`, linked `qwen25-7b-mainloop-1722-3`, saved a project-scoped + preset, and was stopped afterward to cap spend. The backing run is terminated. + +Saved preset: + +```text +path: ~/.dstack/server/projects/endpoint-e2e/presets/qwen-qwen2-5-7b-instruct.dstack.yml +recipe: cb373545 +scheduling cpu=2.. mem=8GB.. disk=100GB.. gpu=nvidia:24GB..48GB:1 +tested: cpu=9 mem=50GB disk=100GB gpu=A5000:24GB:1 +image: vllm/vllm-openai:v0.24.0 +``` + +No-spend reuse preview: + +```text +config: qwen25-7b-reuse-preview.dstack.yml +policy: reuse +selected: Qwen/Qwen2.5-7B-Instruct recipe cb373545 +offers: 12 matching offers under endpoint-e2e-gpu / max_price $0.5 +submitted: no +``` + +Important observations: + +- The agent made a reasonable hardware decision for a 7B model and preserved the broad scheduling + envelope separately from exact A5000 validation hardware. +- The loop recovered from both no-capacity and a concrete runtime/JIT failure without server-side + special cases. +- The first full service attempt still installed/used a runtime path that led to avoidable JIT + friction. For vLLM OpenAI services, the prompt/skill should bias toward pinned official vLLM + serving images unless there is a specific reason to prototype package installation. +- The endpoint worker correctly waited for the agent's functional verification report before + marking the endpoint running. +- Reuse planning is proven for this learned 7B preset; paid reuse deployment was not submitted in + this step. + +## 2026-07-07 Qwen3.6-27B main-loop validation + +Goal: push the learning loop beyond small Qwen models and verify that the agent can use current +recipe sources, choose a defensible 80GB single-GPU shape, wait through a long model startup, perform +real model API verification, and save a reusable recipe. + +Config was kept outside the repo: + +```text +~/dstack-endpoints-demo/qwen27-2026-07-07/qwen36-27b-create.dstack.yml +``` + +Endpoint: + +```yaml +type: endpoint +name: qwen36-27b-mainloop +model: Qwen/Qwen3.6-27B + +preset_policy: create +fleets: + - qwen27-runpod-a100 +spot_policy: auto +max_price: 1.6 +``` + +Result: + +- Endpoint reached `running`. +- Final service run: `qwen36-27b-mainloop-1`. +- Run id: `34ed301f-f1f3-4f8d-bebc-5ecae99568b6`. +- Backend/hardware: RunPod `CA-MTL-3`, NVIDIA A100 80GB PCIe, 31 CPU, 117GB RAM, 200GB disk. +- Price: `$1.39/hr`, within endpoint `max_price: 1.6`. +- Image: `vllm/vllm-openai:v0.24.0`. +- Service resource envelope saved in the recipe: `gpu: nvidia:80GB:1`, `disk: 200GB`. +- Preset list now shows model `Qwen/Qwen3.6-27B` with GPU `nvidia:80GB:1`. +- The endpoint was stopped after verification to cap spend; the backing run terminated cleanly with + final observed cost about `$0.3233`. + +Agent evidence: + +- Read Hugging Face config and vLLM recipe sources for the exact model. +- Identified `Qwen3_5ForConditionalGeneration`, BF16, hybrid GDN linear-attention + full-attention, + multimodal vision encoder, and the official vLLM recipe floor. +- Chose BF16 TP1 on 80GB instead of FP8/NVFP4 because the allowed fleet was A100 and the recipe's + FP8/NVFP4 paths are not the right Ampere assumption. +- Submitted the service directly and recorded why: RunPod is container-style, a task probe would not + reliably preserve the 50GB+ model download for the final service, and the main remaining unknowns + were visible in service startup logs. +- Waited through weight download, model loading, torch compile, warmup, CUDA graph capture, API server + startup, and dstack service probe success. +- Verified the model through the OpenAI-compatible chat API. The successful request returned HTTP 200, + response model `Qwen/Qwen3.6-27B`, `finish_reason=stop`, and content + `The capital of France is Paris.` + +Runtime evidence from logs and verification artifacts: + +- Weight download took about 129 seconds. +- Model loading took 51.1 GiB GPU memory. +- Available KV cache was 18.56 GiB, with 281,804 KV-cache tokens and reported maximum concurrency + 8.60x at 32,768 tokens. +- vLLM used Triton/FLA GDN linear-attention, FlashAttention v2, and FlashInfer sampling without CUDA + or OOM errors. + +Harness observations: + +- The `progress` helper materially improved endpoint logs. The user can now see recipe/hardware + reasoning, why service-first was chosen, when the run landed capacity, when model loading passed, + when API verification started, and when the final report was written. +- The prompt is still long, but the agent followed the core constraints: existing fleet only, + endpoint run names as `-`, final functional verification, and + project-scoped preset save. +- One artifact-writing bug appeared: the final report text had `.39/hr` instead of `$1.39/hr` because + the agent wrote JSON via an unquoted shell heredoc, allowing shell expansion of `$1`. The prompt and + `dstack-prototyping` skill now require Python serializers or quoted heredocs such as `<<'EOF'` for + artifacts. +- The agent still used `head -c` once during source inspection despite the prompt. It did not break + the run, but the harness should continue pushing source inspection toward parsed/bounded fields. +- This run does not prove the task/dev-environment-first path because the allowed fleet was + container-style RunPod. We still need a VM/SSH-fleet scenario where a task or interactive experiment + should clearly be cheaper and more informative than repeated services. + +No-spend reuse preview after stopping: + +```text +config: qwen36-27b-reuse-preview.dstack.yml +policy: reuse +selected: Qwen/Qwen3.6-27B recipe 4a0dd437 +offers: 14 matching RunPod A100 80GB offers under qwen27-runpod-a100 / max_price $1.6 +submitted: no +``` + +Checks run after the prompt/skill updates: + +```text +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +``` + +Result: + +```text +31 passed +``` + +## Probe Task Shape Guard + +The `qwen25-probe-quality-*` live tests showed that prompt wording alone did not reliably produce +the intended interactive task path. One run skipped the task and optimized back to RunPod service +first; the next selected a task and JarvisLabs, but encoded the entire serving investigation as one +batch command. That proved the useful next fix was not more prose, but a structural guard. + +Implemented guard: + +- Endpoint agent task probes must be long-lived attach/SSH targets, normally `commands: + [sleep infinity]`. +- The wrapper rejects batch probe commands that contain the serving investigation inside task YAML: + host checks, Python/framework imports, vLLM/SGLang startup, or curl/API probes. +- The agent should run those checks after attaching/SSHing into the live task. + +Checks after the guard: + +```text +uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +33 passed + +uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +119 passed, 50 skipped + +uv run ruff check src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/services/endpoints/test_claude_agent.py +All checks passed +``` + +Next live e2e should specifically check whether Claude reacts to the wrapper correction by creating +a long-lived task, attaching/SSHing into it, running real serving checks inside, and only then +promoting a clean service. + +## Backend Docs Gate And Endpoint Log Watch + +Config: + +```text +project: endpoint-agent-reasoning +config: qwen25-docs-gate-e2e.dstack.yml +model: Qwen/Qwen2.5-0.5B-Instruct +fleet: probe-quality-guard +policy: create +``` + +Result: + +- Claude classified backends against `https://dstack.ai/docs/concepts/backends.md`, chose JarvisLabs + as VM-based, and rejected RunPod as container-based/no instance volumes. +- The task config used `commands: [sleep infinity]`, `fleets: [probe-quality-guard]`, + `backends: [jarvislabs]`, `gpu: nvidia:16GB..`, and optional Hugging Face instance cache. +- Claude attached/SSHed into the task, started vLLM inside it, and verified a local + `/v1/chat/completions` request. +- Claude stopped the task, waited for the instance to become idle, submitted the same recipe as a + service on JarvisLabs, and verified `/v1/chat/completions` through the dstack service URL. +- The endpoint reached `running`, linked `qwen25-docs-gate-e2e-2`, wrote a schema-clean + `final_report.json`, and was then stopped cleanly. + +CLI/log UX change validated by this run: + +- `dstack endpoint logs -w` was added. +- Foreground endpoint apply now polls endpoint status and streams the same endpoint progress log + stream without using `attach`. +- Endpoint logs print local timestamps because they are progress events, unlike raw streamed run logs. +- Endpoint log following uses an overlap plus duplicate counts, so records sharing the same + timestamp are not swallowed at poll boundaries. + +Checks: + +```text +uv run pytest src/tests/_internal/cli/commands/test_endpoint.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q +42 passed + +uv run ruff check src/dstack/_internal/cli/commands/endpoint.py src/dstack/_internal/cli/services/configurators/endpoint.py src/dstack/_internal/cli/services/endpoint_logs.py src/tests/_internal/cli/commands/test_endpoint.py +All checks passed +``` + +Remaining issue: + +- Endpoint logs are now much more useful, but the first `dstack apply` in this run was started + before the foreground-log-streaming CLI change, so the new attached-apply UX still needs a direct + fresh smoke after this patch. diff --git a/endpoint-implementation-plan.md b/endpoint-implementation-plan.md index fe4fa8f258..a835dbddad 100644 --- a/endpoint-implementation-plan.md +++ b/endpoint-implementation-plan.md @@ -16,7 +16,7 @@ The preset path and the agent path must converge. If the agent can make a model Non-negotiable project invariants: - A preset is a verified deployment artifact, not just a service YAML. -- `resources` are scheduling requirements used for reuse/offer matching; `tested_resources` are exact verified hardware evidence. +- Each preset is keyed by model and contains one or more deployment recipes. A recipe's `service` contains the scheduling resources used for reuse/offer matching; `validations` store exact verified hardware evidence grouped in the same order as the service replica groups. - The server owns endpoint state, locking, run linking, stopping/cleanup, and preset persistence. The agent owns deployment investigation and final functional verification. - The server must not mark an agent-created endpoint running based only on its own service-readiness bookkeeping; the agent must report that the requested model was actually served and answered a real model request. - Work should be driven by real deployment failures. Avoid adding abstractions before live runs show they are needed. @@ -38,10 +38,11 @@ Keep this section current while implementing. It is the short operational view o - [x] Endpoint apply treats any terminal same-name endpoint as finished: no stop prompt; create resets the same endpoint row back to `submitted`, and only non-terminal same-name endpoints require stop/override. - [x] Preset-backed endpoint submission handles its deterministic service-run-name conflicts like run apply: non-terminal conflicting runs fail before submission, while terminal conflicting runs are left to the existing run submission path to recycle. - [x] Endpoint readiness uses backing service readiness only: run is `RUNNING`, has a registered running job, and exposes `ServiceSpec.model.base_url`; no extra endpoint probe in v1. -- [x] Preset storage/parsing: `EndpointPresetService`, local-dir implementation, `endpoint-preset` YAML wrapper, ordered `replica_spec_groups`, validation, atomic no-overwrite save, env value redaction, and literal non-secret env preservation. -- [x] Learned preset plumbing: build a preset from a ready service run, record replica spec groups in service order, preserve the number of currently registered running replicas per group, and save it when an agent-provisioned endpoint becomes running. -- [x] Endpoint preset CLI: `dstack endpoint preset list|get --json|delete`. The compact list shows scheduling resources used for reuse/offer matching; exact verified hardware stays in `tested_resources` and is exposed by `get --json`, not the compact list. -- [x] Endpoint status names adjusted: `clauding` while the server agent investigates/deploys, `running` for a ready endpoint, `stopping`/`stopped` for endpoint stop. The runtime `active` alias was removed; local prototype DB rows should be cleaned directly. +- [x] Preset storage/parsing: `EndpointPresetService`, project-local implementation under `/projects//presets`, `endpoint-preset` YAML wrapper, model-keyed `recipes`, ordered `validations`, legacy `replica_spec_groups` conversion, validation, merge-by-model list/get/save/delete, env value redaction, and literal non-secret env preservation. +- [x] Learned preset plumbing: build or merge a model-level preset from a ready service run, preserve the service recipe/resources as the scheduling contract, record exact running replica resources in `validations`, and save it when an agent-provisioned endpoint becomes running. +- [x] Endpoint preset CLI: `dstack endpoint preset list|get --json|delete`. The compact list groups by model and shows only the scheduling GPU; `-v` shows full service scheduling resources. Child `recipe=N` rows appear only when a model has multiple recipes; internal recipe ids and exact verified hardware stay in `get --json`. +- [x] Endpoint list UX now follows the `ps` convention more closely: default output shows unfinished endpoints, or the latest finished endpoint only when nothing is active; `POLICY` is shown by default and the backing service run is only shown in verbose/detail/JSON output. +- [x] Endpoint status names adjusted: `prototyping` while the server agent investigates/deploys, `running` for a ready endpoint, `stopping`/`stopped` for endpoint stop. The runtime `active` alias was removed; local prototype DB rows should be cleaned directly. - [x] Endpoint UX split from run UX: `dstack endpoint logs` reads endpoint progress logs, `dstack endpoint stop` stops endpoints, top-level `dstack logs`/`dstack stop` remain run-only, and top-level `dstack preset` was removed. - [x] Running endpoint apply output renders known relative proxy URLs as absolute URLs using the configured API server URL. - [x] Endpoint migration is safe for Postgres partial indexes and boolean server defaults. @@ -53,21 +54,64 @@ Keep this section current while implementing. It is the short operational view o - [x] Raw Anthropic Messages-loop prototype was removed. It was the wrong abstraction; the endpoint agent must use a real agent runtime, not a hand-rolled tool loop. - [x] Packaged endpoint-agent prompt/context resources: `resources/system_prompt.md` plus repo-root `skills/dstack` and `skills/dstack-prototyping`, force-included into wheels/sdists and copied into each Claude workspace. - [x] First full learning-loop proof: Claude-created Qwen preset was reused later by `preset_policy: reuse` without invoking Claude; the endpoint reached `running`, answered a model request, and stopped cleanly. +- [x] Fresh create-policy Claude run: `Qwen/Qwen2.5-0.5B-Instruct` on RunPod reached `running`, verified a real OpenAI-compatible chat request, saved preset `qwen-qwen2-5-0-5b-instruct-e30ddb94`, and stopped the backing run cleanly. +- [x] No-cost reuse preview for the fresh Qwen2.5 preset selected the learned preset and found 5 matching RunPod offers under the endpoint constraints. +- [x] Paid reuse deployment for the fresh Qwen2.5 preset reached `running` without Claude, answered a model request on RunPod A5000, and stopped cleanly. +- [x] Fresh create-policy rerun after prompt/env fixes exposed a real harness weakness: the agent stopped using `~/.dstack/config.yml` and append-only submission records worked, but it repeatedly submitted full services on a container backend to learn runtime facts that should be prototyped with a dev environment or small task when an allowed reusable VM/SSH fleet exists. +- [x] Stop-time agent cancellation implemented after the live leak: `dstack endpoint stop` now asks the agent runtime to abort, terminates the stored Claude process group, keeps the endpoint `stopping` if the agent process is on another host, and still stops linked/submitted runs through `EndpointRunSubmissionModel`. +- [x] Stop-time cancellation live validation: stopping before Claude starts and stopping after Claude starts both reached `stopped` with no endpoint-submitted runs left. A zombie-only process-group issue was found and fixed during validation. +- [x] Historical create-policy run in an isolated project with no fleets proved the real agent loop could deploy and verify a model, but also showed an unwanted contract: Claude created an endpoint-scoped fleet. That behavior is now rejected; fleet creation is a user/admin step before endpoint submission. +- [x] No-fleet contract validation: with no active fleet, endpoint apply now shows "The project has no fleets. Create one before submitting an endpoint." for any `preset_policy`; forced detached apply exits `1` and does not resubmit a terminal endpoint. The pipeline also fails already-submitted endpoints with the same no-fleets status before preset or Claude processing. +- [x] Existing-fleet contract validation: after creating zero-target fleet `endpoint-agent-reasoning-fleet`, the endpoint plan returned the agent path. Claude started, inspected the existing fleet, correctly ignored prior stopped run `qwen25-7b-vm-reasoning-1` as old/different-fleet work, and used `dstack offer --fleet endpoint-agent-reasoning-fleet ...`. The run was stopped before any new service submission or GPU spend. +- [x] Endpoint progress stream no longer prescribes labels/categories/templates: the server accepts plain text progress lines or JSON objects with a `message` string and displays only the message text. +- [x] Harder existing-fleet e2e exposed prompt-only enforcement failure: Claude correctly used `dstack offer --fleet endpoint-agent-reasoning-fleet`, then wrote a probe task without `fleets` and submitted it. The task failed with `no offers` before provisioning (`cost: 0.0`), and the endpoint/agent were stopped. A workspace-local `dstack` CLI guard now blocks fleet apply and blocks agent-submitted run configs that omit or violate explicit endpoint `fleets`. +- [x] Harder existing-fleet e2e after the CLI guard succeeded end-to-end: `Qwen/Qwen2.5-7B-Instruct` deployed on existing fleet `endpoint-agent-reasoning-fleet`, selected Jarvis L4:24GB at `$0.44/hr`, served through `vllm/vllm-openai:v0.24.0`, answered a real `/v1/chat/completions` request, saved project preset `qwen-qwen2-5-7b-instruct-d79aa1d8`, and stopped cleanly. The test fleet was deleted afterward to stop idle spend. +- [x] Guard behavior validated in the live run: Claude first wrote a service config without `fleets`, the workspace-local `dstack` shim rejected it, Claude corrected the YAML to include `fleets: [endpoint-agent-reasoning-fleet]`, previewed/applied it, and the final saved preset remained usable without Claude. +- [x] Learned-preset reuse preview validated for the 7B run: while the original service occupied the only `nodes: 0..1` fleet slot, reuse correctly matched the preset but showed no offers; after stopping the endpoint, the same preview selected the preset and showed the idle Jarvis L4 offer without invoking Claude. +- [x] Agent CLI guard now enforces the default "all existing project/imported fleets" path by passing discovered active usable fleet names into the workspace shim, and rejects `dstack offer` / `dstack apply` calls that omit allowed fleets or violate accepted endpoint profile constraints such as backend, region, instance type, spot policy, max price, durations, instances, tags, and backend options. +- [x] Fresh RunPod existing-fleet e2e after the guard: `Qwen/Qwen2.5-1.5B-Instruct` with `preset_policy: create`, `backends: [runpod]`, `fleets: [endpoint-e2e-runpod]`, `spot_policy: on-demand`, and `max_price: 0.5` reached `running`. Claude first omitted `fleets` in the service YAML, the workspace shim rejected the preview, Claude corrected the config, the first service submission failed with transient RunPod `failed_to_start_due_to_no_capacity`, the second submission added retry, landed RTX 2000 Ada at `$0.24/hr`, answered `/v1/chat/completions`, saved preset `qwen-qwen2-5-1-5b-instruct-4d68b0a3`, and `dstack endpoint stop` terminated the backing run cleanly. +- [x] Immediate reuse preview for that saved Qwen2.5-1.5B preset validated both capacity states: while the only allowed RunPod fleet slot was occupied, the plan selected the preset but showed no offers; after stopping the endpoint, the same reuse preview selected the preset and showed 7 matching offers without invoking Claude. +- [x] Prompt/skill tightened from the fresh trace: final service YAML must carry applicable endpoint constraints, progress must report run-capacity/running/final-verification milestones, polling/probe commands must stay bounded to avoid Claude Code Bash timeouts, and `final_report.json` must stay limited to the JSON-schema fields while hardware/driver/reasoning evidence lives in workspace artifacts. +- [x] Fresh validation run after prompt/skill tightening: `Qwen/Qwen2-0.5B-Instruct` with `preset_policy: create`, `backends: [runpod]`, `fleets: [endpoint-e2e-runpod]`, `spot_policy: on-demand`, and `max_price: 0.5` reached `running`, answered `/v1/chat/completions`, saved preset `qwen-qwen2-0-5b-instruct-532ddf05`, and stopped cleanly. Improvements were real: the first service YAML included `fleets`, endpoint logs included running and verification milestones, polling was bounded instead of concurrent watchers, and `final_report.json` was schema-clean. Remaining harness/preset quality issues from the same trace: the service requested `gpu: 1` without a GPU-memory floor, wrote a transient `"run_id":"pending"` submission record, waited on a Uvicorn log marker after an early proxy probe, and still did not capture a direct `nvidia-smi` driver string. +- [x] Immediate reuse preview for the saved Qwen2-0.5B preset validated both capacity states: while the only allowed RunPod fleet slot was occupied, the plan selected the preset but showed no offers; after stopping the endpoint, the same reuse preview selected the preset and showed 7 matching offers without invoking Claude. +- [x] Removed the premature agent-budget surface from v1: no endpoint `max_agent_budget`, no `DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD`, no DB/session spend fields, no CLI plan row, no API plan `max_budget`. Cost/budget governance is Later and must be added only with durable per-session accounting. +- [x] Tightened the endpoint system prompt and `dstack-prototyping` skill after the `gpu: 1` trace: final LLM services should normally request a defensible GPU memory/count envelope, and count-only GPU scheduling must be intentional and recorded in hardware reasoning. +- [x] Agent session restart semantics tightened: a server restart or dead same-host Claude process resumes the same endpoint agent session/workspace instead of creating a new session; a new endpoint lifecycle from re-apply creates the next session and writes `previous_sessions.md` so the agent can use older session evidence read-only. +- [x] Same-host restart and minimum-resource guidance validated in a fresh paid RunPod run: `Qwen/Qwen2-0.5B-Instruct` stayed in the same agent session across a local server restart, did not spawn a duplicate Claude process, deployed `qwen2-05b-restart-1110-1`, verified `/v1/chat/completions`, saved a model-level preset recipe, and stopped cleanly. The final service YAML used `gpu: 16GB..24GB:1`; exact RTX 3090 placement belongs under recipe `validations`. +- [x] Endpoint agent stdout is no longer copied to endpoint logs. Endpoint logs are populated from `progress.jsonl` and explicit server lifecycle messages; Claude assistant chatter and final markdown summaries remain in trace/artifacts. +- [x] Endpoint agent logs now start promptly: the server writes an immediate progress line when a detached Claude session starts, stores it in `progress.jsonl`, advances the progress offset to avoid replay, and the prompt requires Claude's first workspace action to be a short progress message before inspecting state/offers/docs. Live smoke `qwen25-log-start-smoke` showed the startup line and Claude's first progress line, then was stopped before any service submission. +- [x] Follow-up paid Qwen2.5 preset-reuse deployment validated the no-Claude path after the model-level recipe format: `Qwen/Qwen2.5-0.5B-Instruct` with `preset_policy: reuse` selected recipe `4a73893b`, deployed on RunPod A5000, reached `running`, answered `/v1/chat/completions`, and stopped cleanly. +- [x] Local duplicate preset files are no longer a planning/list/get inconsistency: the local service merges files by model in memory, `get --json` returns all recipes for the model, `save` collapses duplicates into one canonical file, and `delete` removes all files for that model. +- [x] Broad-fleet vendor-aware harness check: endpoint config used neutral fleet `endpoint-e2e-gpu` with Lambda/Verda/JarvisLabs/RunPod and no endpoint `backends`; current matching offers under `$0.5/hr` were still all RunPod, Claude selected RunPod based on offers, wrote final service resources as `gpu: nvidia:16GB..24GB:1`, verified `/v1/chat/completions`, saved a vendor-aware recipe, and stopped cleanly. Reuse preview after freeing the `nodes: 0..1` fleet slot showed offers without invoking Claude. +- [x] Prompt/guard hardening after repeated YAML mistakes: endpoint prompt and `dstack-prototyping` now distinguish plural service YAML fields (`fleets`, `backends`, `regions`, `instance_types`) from singular CLI flags (`--fleet`, `--backend`, `--region`, `--instance-type`), and the workspace CLI guard returns actionable errors for singular run-config keys before a paid preview/apply can proceed. +- [x] Existing no-spend endpoint functionality e2e: endpoint list/get/logs/stop, endpoint-vs-run command separation, preset list/get, reuse/create/no-fleet apply previews, and preset verbose output. Fixed `dstack endpoint preset -v list` so parent `-v` is not overwritten by the child parser. +- [x] Main learning-loop validation after prompt/guard fixes: `Qwen/Qwen2.5-0.5B-Instruct` with `preset_policy: create` recovered from a real RunPod CUDA driver mismatch, submitted a pinned CUDA-12 vLLM image, verified `/v1/chat/completions`, linked the endpoint to the final run, saved a preset, stopped cleanly, and a subsequent `preset_policy: reuse` endpoint selected the freshly verified recipe, reached `running` without Claude, then stopped cleanly. +- [x] Learned recipe priority fixed: `save_preset` now writes incoming learned recipes before older recipes for the same model, preserving the simple "first recipe with offers wins" planner while ensuring the recipe just proven by the agent is what immediate reuse sees. +- [x] Harder 7B learning-loop validation: `Qwen/Qwen2.5-7B-Instruct` with `preset_policy: create` selected a NVIDIA 24GB..48GB envelope, recovered from no-capacity and a FlashInfer/nvcc runtime failure, deployed `vllm/vllm-openai:v0.24.0` on RunPod RTX A5000, verified `/v1/models` and `/v1/chat/completions`, saved a project-scoped recipe with exact A5000 validation evidence, stopped cleanly, and a no-spend `preset_policy: reuse` preview selected the learned recipe with matching offers without invoking Claude. +- [x] Larger-model learning-loop validation: `Qwen/Qwen3.6-27B` with `preset_policy: create` used current HF/vLLM recipe evidence, chose BF16 TP1 on `gpu: nvidia:80GB:1`, deployed `vllm/vllm-openai:v0.24.0` on RunPod A100 80GB at `$1.39/hr`, waited through 51.1 GiB model loading and CUDA graph capture, verified `/v1/chat/completions`, linked `qwen36-27b-mainloop-1`, and saved a project-scoped preset. The endpoint was stopped cleanly after verification, and a no-spend `preset_policy: reuse` preview selected the saved recipe with 14 matching offers without invoking Claude. This proved the real agent loop can handle a non-trivial current model, not just small smoke models. +- [x] Qwen3.6-27B trace hardened artifact-writing guidance: the agent verified the service correctly, but one final-report string was corrupted from `$1.39/hr` to `.39/hr` by an unquoted shell heredoc. The endpoint system prompt and `dstack-prototyping` skill now require Python serializers or quoted heredocs such as `<<'EOF'` for JSON/Markdown/YAML artifacts. +- [x] Backend-placement/task-first validation: `Qwen/Qwen2.5-7B-Instruct` on fleet `reusable-vs-container` had cheaper RunPod container offers and a JarvisLabs L4 reusable/inspectable offer. Claude chose JarvisLabs despite the slightly higher hourly price, submitted a task first, then promoted a final service that reused the warm L4 instance and verified `/v1/models` plus `/v1/chat/completions`. The endpoint was stopped and the temporary fleet deleted. This proves task-first on a reusable backend, but not the interactive SSH/dev-environment loop. +- [x] Probe-quality correction from the same validation: the task probe observed `nvidia-smi` and host driver evidence but failed on `python` missing before proving framework/runtime/server behavior. The final service still proved the endpoint, but the prompt and `dstack-prototyping` now state that host sanity checks are not recipe proof; a probe must exercise the intended serving stack before service promotion. +- [x] Probe-quality rerun exposed an earlier decision failure before it could test the probe fix: Claude saw RunPod and JarvisLabs offers but inferred "both backends are container-only (no reusable VM/SSH state)", skipped the task/dev probe, and submitted a direct RunPod service. We stopped the endpoint, terminated the service, and deleted the temporary fleet. Prompt/skill now forbid inferring "container-only" or "no reusable state" from offers alone and require uncertainty to be resolved, not treated as proof that a probe cannot help. +- [x] Second probe-quality rerun improved placement and run type but exposed a sharper harness miss: Claude chose JarvisLabs L4 and submitted a task first, but encoded the full investigation as one batch command chain and omitted all optional instance cache mounts (`configuration.volumes=[]`, `runtime.volume_names=[]`). We stopped the endpoint, terminated the task, deleted the fleet, and tightened prompt/skill so attach/SSH-capable probes should be long-lived interactive tasks/dev environments (`sleep infinity` or equivalent) and Hugging Face-style model-serving probes/services should use optional cache mounts or record why they were omitted. +- [x] Third probe-quality rerun showed the limit of prompt-only enforcement: Claude added the optional Hugging Face cache mount, but still generated a batch task instead of a long-lived attach/SSH probe, again wrote `jarvislabs+runpod (container-style)`, and submitted RunPod L4. We stopped the endpoint/run immediately and deleted the fleet. Next fix should be structural: server-generated backend/fleet capability context and possibly a pre-submit guard/required artifact for interactive-probe decisions, not more prose. +- [x] Backend docs-gated rerun validated the intended placement/prototyping path: `Qwen/Qwen2.5-0.5B-Instruct` on fleet `probe-quality-guard` classified JarvisLabs as VM-based from backend docs, rejected RunPod as container-based/no instance volumes, submitted a long-lived task with `sleep infinity`, attached/SSHed into it, verified vLLM locally, stopped the task, reused the idle JarvisLabs L4 for the service, verified `/v1/chat/completions` through the dstack service URL, reached `running`, and stopped cleanly. CLI UX now supports `dstack endpoint logs -w`, and foreground endpoint apply streams the endpoint progress log stream while polling status without `attach`. - [x] Focused endpoint tests and static checks currently pass (`pytest` endpoint suites, `ruff`, `pyright`). ### Still open for v1 -- [ ] Harden the real Claude Code subprocess runtime based on live failures: restart/resume semantics, stop-time cancellation, duplicate-process prevention, packaging, and live-run efficiency. -- [ ] Finish runtime durability: persisted usage accounting, restart-safe handoff from agent report to linked service run, endpoint stop abort handling, and cleanup of non-final candidate runs. -- [ ] Guard learned-preset quality: scheduling `resources` must stay broad enough for reuse, exact hardware must remain in `tested_resources`, and saved presets must be usable without the agent. -- [ ] Wire structured agent handoff into preset provenance: final candidate run name/id, final service YAML/content, recipe sources, verification summary, and failure summary. +- [ ] Harden the real Claude Code subprocess runtime based on live failures: multi-host restart behavior, duplicate-process prevention across server instances, packaging, and live-run efficiency. +- [ ] Finish runtime durability: final-report handoff after process/server crashes, multi-host reconciliation, and cleanup of non-final submitted runs after restart. +- [ ] Guard learned-preset quality: recipe service `resources` must stay broad enough for reuse but not too loose for correctness, exact hardware must remain in `validations`, and saved presets must be usable without the agent. +- [ ] Add server-generated backend/fleet capability context for the agent workspace: allowed fleets, current nodes/idle instances, backend capability facts known to dstack, whether attach/SSH/dev environments are expected to work, and whether instance cache/volumes are plausible. The last probe-quality run showed that asking the agent to infer this from docs/offers is unreliable. +- [ ] Add a lightweight pre-submit guard or required pre-submit artifact for create-recipe probes: when the chosen experiment is a task/dev probe and attach/SSH is available, the artifact must state whether it will be long-lived/interactive, whether cache mounts are included, and why batch commands are acceptable if used. +- [ ] Run a true interactive SSH/dev-environment scenario on reusable capacity. The JarvisLabs validations proved task-first and better placement, but not the intended long-lived attach/SSH loop, interactive command tuning, optional instance cache mounts, or stronger local server probing before promotion. +- [ ] Decide whether any lightweight preset metadata is worth adding later. Do not add final run ids, recipe sources, or verification summaries to the YAML until a concrete reader uses them. - [ ] Runtime recipe grounding against vLLM recipes / SGLang docs / HF model cards, with mocked zero-network tests. -- [ ] Foreground endpoint apply following server-side endpoint logs/status without attach. -- [ ] Endpoint preset inspect polish: keep `dstack endpoint preset get --json` useful enough for exact `tested_resources`, provenance comments/metadata, and service recipe review without cluttering the compact list output. +- [ ] Endpoint preset inspect polish: keep `dstack endpoint preset get --json` useful enough for exact `validations` and service recipe review without cluttering the compact list output. - [ ] Real agent runtime dependencies installed automatically by normal server install/deploy paths: local `uv` server installs and server Docker images must include the runtime; no manual post-install dependency step. - [ ] Endpoint update/version design before in-place updates: a `configuration_version`/deployment guard so stale background workers in multi-server Postgres setups cannot mark a newer endpoint config running. - [ ] Documentation: endpoint reference schema page, env vars, concepts page, manual e2e runbook, example presets. -- [ ] Real no-preset agent e2e with a small model and budget-confirmed hardware. ### Critical assessment @@ -75,23 +119,41 @@ The endpoint storage/status/preset/run-link side is becoming reasonable. The fea Highest current risks: -- Agent works once but saves a preset that is too exact, stale, missing provenance, or not reusable. -- Agent wastes budget because hardware selection, offer inspection, or failure recovery is weak. +- Agent works once but saves a preset recipe that is too exact, stale, missing validation evidence, or not reusable. +- Agent wastes budget because hardware selection, offer inspection, experiment choice, or failure recovery is weak. - Agent logs/traces are either too noisy for users or too thin for debugging. -- Server restart/stop during `clauding` can leave ambiguous state or orphan candidate runs. -- Duplicate learned presets accumulate without a repair/update policy. +- Multi-host server restart during `prototyping` can leave ambiguous state or orphan submitted runs if the new server cannot observe the old host's Claude process/workspace. +- Multiple learned recipes for the same model accumulate without an explicit update policy. - We overfit code around the first Qwen smoke runs before testing enough real model/backends/failure modes. +- One fresh Claude run used macOS-specific `sed -i` and read `~/.dstack/config.yml` to find URL/token for verification; both were prompt/context defects tightened afterward. +- A later fresh Claude rerun fixed those mechanics but failed the higher-level harness test: after service failures caused by FlashInfer runtime JIT, a RunPod provisioning stall, and CUDA 13 package/driver mismatch, the agent kept submitting services instead of switching to a better prototyping path when possible. It also exposed a stop bug: the endpoint became `stopped` while the Claude process continued and submitted another paid run. Process-group abort is now implemented and live-validated; the validation also fixed a zombie-only process-group edge case. +- The latest isolated no-fleet run passed mechanically and showed useful backend/fleet choice, but endpoint logs are still too sparse for watching a real investigation. The next improvement should make the agent write more natural-language decision updates without reintroducing hardcoded labels or templates. +- The latest isolated run also showed that fleet choice needs explicit evidence. `gpu:16GB:1`, `nodes: 0..1`, max price, and idle duration were plausible for a tiny Qwen model, but the harness must make the agent justify which existing fleet/resource envelope it uses without editing fleet configuration. +- Prompt instructions alone are not enough for the fleet/profile contract. The workspace-local CLI guard now enforces allowed fleets and common accepted profile constraints, but this must stay tied to real traces; if we accept more endpoint fields, they must either be enforced by the guard/submitted YAML or explicitly rejected/deferred. +- The 7B existing-fleet run succeeded, but also exposed real harness inefficiency: Claude spawned several concurrent shell polling loops instead of one bounded status loop. This is not just cosmetic; on long deployments it can waste process slots, tokens, and make command traces hard to read. +- Claude's first model API probe used a 420s internal deadline but was killed by Claude Code's 2-minute Bash tool timeout. The agent recovered by checking logs and retrying, but the harness should teach short timed probes and background polling explicitly instead of relying on recovery. +- The service proxy briefly returned `404 Service not found` immediately after service readiness. Claude retried and succeeded, so this is likely a proxy/registration timing edge, but the prompt/runbook should treat early 404s as retryable only after run JSON and service probes indicate the service is registered. +- The fresh Qwen2.5-1.5B RunPod run succeeded mechanically, but still showed prompt/harness gaps: Claude initially omitted the required `fleets` field and depended on the guard to catch it; it used a long polling command that hit the Claude Code Bash timeout; endpoint progress did not include the final "service running" and "model verified" milestones; and the handwritten `final_report.json` contained extra fields with malformed strings. The server ignored the extra fields and used the required schema fields, but future traces should be cleaner. +- The fresh RunPod run did not capture a direct `nvidia-smi` driver string. The service logs proved the cu129 vLLM image ran successfully with CUDA/FLASH_ATTN, which is useful evidence, but validation artifacts should distinguish "runtime worked" from "host driver version recorded." +- The fresh Qwen2-0.5B validation run showed that the latest prompt tightening helped materially, but it also exposed the opposite preset-resource risk. The saved preset is reusable, but because the final service YAML used `gpu: 1`, the recipe service resources had no GPU-memory floor even though validations prove RTX 2000 Ada 16GB. This is not necessarily wrong for a 0.5B model, but the harness needs a sharper rule for when count-only GPU constraints are acceptable versus when a memory envelope is required. +- The fresh Qwen2-0.5B validation run still used logs as part of readiness after an early proxy miss ("wait for Uvicorn running"). That was not fatal because the agent had already checked run JSON and later sent a real model request, but the intended loop is still: run JSON for lifecycle, model API probe for proof, logs for diagnosis/explanation. +- The next fresh Qwen2-0.5B run fixed the resource-envelope issue: the final service YAML used `gpu: 16GB..24GB:1` and the learned recipe kept exact RTX 3090 placement under `validations`. This is a good sign, but not enough to generalize to larger models or multi-replica deployments. +- The same run validated same-host server restart while Claude was running: the server restarted, reused the same agent session/workspace, and did not start a duplicate Claude process. Multi-host restart and Postgres multi-server coordination remain unproven. +- The same run also showed that the workspace CLI guard is necessary: Claude first used `backend` instead of `backends` and omitted `fleets`; the guard blocked preview before spend and Claude corrected the YAML. Treat guard catches as useful safety, but still reduce repeated prompt-level mistakes. +- Endpoint logs leaked Claude assistant stdout even though `progress.jsonl` was clean. The runtime now stops copying assistant stream text to endpoint logs; only progress messages and explicit server lifecycle messages belong there. +- A no-cost reuse preview after the same run selected an older Qwen2 recipe rather than the new resource-envelope recipe. The local store now merges duplicate files into one model-level preset. V1 keeps selection deterministic: try stored recipes in order and use the first one with available offers under the endpoint's effective fleets/profile constraints. Explicit user recipe selection stays Later. +- The 7B main-loop run is a better signal than the tiny Qwen runs: Claude chose a reasonable NVIDIA 24GB..48GB envelope, recovered from no-capacity, diagnosed a FlashInfer/nvcc failure, and produced a reusable recipe. It still showed that the harness should prefer pinned official vLLM serving images for final vLLM OpenAI services unless runtime package installation is itself the experiment. +- The probe-quality rerun showed that prompt-only backend capability reasoning is still fragile. The agent can overrule the intended reusable/inspectable placement route by inventing a backend-capability conclusion from an offer table. We either need stronger prompt/skill constraints or explicit backend/fleet capability context from the server before the agent decides task/dev vs service-first. ### Immediate next steps: repeat the loop, then harden Do these in order. Do not add more harness surface area until the previous step has produced evidence. -1. **Audit the reused preset after the run.** Confirm the preset's `resources` are scheduling requirements, `tested_resources` are exact evidence, the service recipe is sufficient, and offer matching is not over-narrow. If reuse fails later, fix the preset contract/builder before touching the agent prompt. -2. **Run one fresh no-preset agent deployment with a small model and a confirmed budget.** Capture command count, elapsed time, spend, hardware choice, service YAML, dstack plan output, candidate cleanup, progress logs, final report, and saved preset. -3. **Harden only from observed failures.** Fix concrete problems in prompt/context, endpoint logs, final report validation, preset saving, run linking, or cleanup. Avoid speculative tools/functions. -4. **Add durable agent budget accounting before retries/resumes.** `max_agent_budget` must cap total spend per endpoint provisioning attempt, not just a single subprocess invocation. -5. **Harden restart/stop while clauding.** Stop should abort the live process and stop linked/candidate runs using explicit submission records. Restart should reconcile existing workspace, final report, and submissions before launching anything new. -6. **Move to a more common/interesting model only after the small loop repeats.** Confirm hardware and budget before trying larger models. Use those runs to evolve hardware-selection behavior and recipe grounding. +1. **Keep v1 recipe selection simple and deterministic.** Repeated agent tests now leave multiple provisionable recipes for the same model. Local duplicate files are merged. V1 should try recipes in stored order and use the first recipe whose normal run plan has available offers under the endpoint's effective fleets/profile constraints. Do not add quality ranking yet. +2. **Reduce harness waste exposed by guard catches.** The guard must stay, but the agent should not repeatedly write `backend` instead of `backends` or omit required `fleets` from final service YAML. +3. **Harden final-report handoff and submitted-run reconciliation.** Same-host restart while Claude is live now works, but the endpoint worker still needs stronger tests around final report written before link, process/server crash timing, and cleanup of non-final submitted runs. +4. **Deepen the reusable-capacity route.** The JarvisLabs run proved that the agent can prefer a reusable/inspectable backend over cheaper container offers and start with a task. Next, prove the stronger path: when attach/SSH/dev-environment is useful, the agent keeps an interactive probe alive, checks the intended image/runtime/command, and promotes only after the serving stack is actually exercised. Confirm hardware and expected GPU spend before each paid run. +5. **Keep agent budget/cost governance out of v1.** Add it later only with durable per-session accounting and an actually enforced runtime contract; do not expose a config field or env var before that exists. ### How to overcome the current risks @@ -99,20 +161,25 @@ Treat this as a working hypothesis, not a fixed design. After each real endpoint | issue | realistic mitigation | evidence to collect | reconsider if | |---|---|---|---| -| Learned preset works once but is not reusable | Run an immediate preset-reuse test after every successful agent deployment. Validate that the saved `service` + `resources` can be planned/submitted without the agent and that `tested_resources` are only evidence. | apply plan path selected, offers shown, final service run, endpoint status, model request result, saved preset YAML before/after | the preset path fails for reasons unrelated to transient capacity; then fix preset contract/builder before prompt changes | -| Preset `resources` are too exact or too loose | Keep `resources` sourced from the final service config, but add validation/reporting that flags exact GPU names or full exact CPU/memory/disk constraints in learned presets unless explicitly justified by the service shape. Do not auto-generalize blindly; first inspect the agent's service YAML and the run plan. | final service YAML, saved `resources`, saved `tested_resources`, offer count before/after, reason for any exact GPU/model constraint | exact constraints repeatedly block reuse, or broad constraints repeatedly schedule hardware that cannot serve the model | -| Agent chooses poor hardware or wastes attempts | Give the agent a required decision trace: model sizing evidence, candidate hardware envelope, current dstack offers, planned service resources, and why it is submitting now. Start with prompt/context; only add server-side enforcement after observed bad behavior is clear. | recipe/model sources, offer table excerpt, chosen resources, rejected alternatives, command count, elapsed time, GPU spend | the trace is absent or useless in two real runs; then enforce a structured pre-submit decision artifact before `dstack apply -y -d` | -| Agent logs are noisy or unhelpful | Split logs into two layers: endpoint log gets concise major events authored by the agent; workspace trace stores command output, YAML, raw logs, and final report. Do not put full CLI output or service logs in endpoint logs. | `dstack endpoint logs ` readability, workspace artifacts, ability to diagnose failure without server stdout | endpoint logs still duplicate/replay, hide important state, or force reading server logs for normal debugging | -| Restart during `clauding` duplicates work or loses the final run | Reconcile before launch: check existing process metadata, final report, endpoint run submissions, linked `service_run_id`, and live candidate runs. Do not start a new agent until reconciliation decides the previous attempt is unrecoverable. | restart test with a live process, restart test after final report before link, endpoint submissions, linked run, no duplicate paid process | reconciliation logic becomes fragile or still misses cases; then consider moving agent execution to a separately tracked durable task model | -| Stop during `clauding` leaks resources | Add process cancellation and candidate-run cleanup using `EndpointRunSubmissionModel`, not name heuristics. Endpoint stop should request abort, stop linked/submitted non-terminal runs, then finish through the normal run lifecycle. | stop during Claude process, stop after candidate run submitted, final run statuses, endpoint stopped, no running GPU run | cancellation depends on runtime behavior we cannot rely on; then isolate agent execution in a managed process/task with explicit supervisor state | -| Budget cap can be bypassed by restart/resume | Persist provider usage/spend per endpoint provisioning attempt and check remaining budget before starting another agent process. Until this exists, do not add automatic retries/resumes that can spend again. | Claude reported usage/cost, persisted attempt spend, remaining budget, refusal path when exhausted | Claude Code cannot expose reliable usage data; then use a stricter wall-clock/process cap plus operator-visible warning until runtime changes | -| Duplicate learned presets accumulate | Keep append-only for v1, but add inspection/deletion and a simple duplicate report. Do not auto-update until the harness can prove an older preset is not reproducible under its claimed conditions. | duplicate model/resource list, preset source endpoint/run, reuse success/failure per preset | duplicates make selection unstable; then add deterministic ranking or explicit user-selected preset before adding automatic update | +| Learned preset works once but is not reusable | Run an immediate preset-reuse test after every successful agent deployment. Validate that the saved recipe `service` can be planned/submitted without the agent and that `validations` are only evidence. | apply plan path selected, offers shown, final service run, endpoint status, model request result, saved preset YAML before/after | the preset path fails for reasons unrelated to transient capacity; then fix preset contract/builder before prompt changes | +| Recipe resources are too exact or too loose | Keep resources sourced from the final service config, but add validation/reporting that flags exact GPU names or full exact CPU/memory/disk constraints in learned recipes unless explicitly justified by the service shape. Do not auto-generalize blindly; first inspect the agent's service YAML and the run plan. | final service YAML, saved service resources, saved validations, offer count before/after, reason for any exact GPU/model constraint | exact constraints repeatedly block reuse, or broad constraints repeatedly schedule hardware that cannot serve the model | +| Agent chooses poor hardware/backend or wastes submissions | Give the agent a required decision trace: model sizing evidence, hardware envelope, current fleet-filtered dstack offers, backend/runtime characteristics inside allowed fleets, planned run resources, and why it is submitting now. Hourly price is a constraint, not the objective: cheaper container placement should not beat a slightly more expensive reusable placement if the reusable path reduces total iteration time, repeated downloads, image churn, or debugging risk. Start with prompt/context; only add server-side enforcement after observed bad behavior is clear. | recipe/model sources, offer table excerpt, chosen resources, rejected alternatives, selected backend placement, selected fleet, command count, elapsed time, GPU spend | the trace is absent or useless in two real runs, or the agent picks a container placement while a viable reusable placement existed without a good reason; then enforce a structured pre-submit decision artifact before paid `dstack apply` | +| Agent uses service submissions as every experiment | Make the prompt/skill require an explicit experiment-type choice. If the uncertainty is package/runtime, launch flags, model load, driver/CUDA compatibility, or backend provisioning and a viable VM/SSH/Kubernetes placement exists inside the allowed fleets, the agent should normally start with a task or interactive experiment before the final service. The agent still makes the final call; this is not a hardcoded rule. | run type per submitted run, why that type was chosen, elapsed provisioning time, whether reusable/inspectable placement was available, service vs task/dev command count, final proof | the agent keeps submitting services for non-URL-wiring uncertainty, or skips a useful task probe without a concrete reason; then add a structured pre-submit artifact that must justify service vs task/dev | +| Agent invents backend capability | Do not let the agent infer "container-only", "no reusable state", or "no SSH/dev value" from an offer table alone. It must use fleet state, backend/fleet behavior evidence, or mark capability as unknown and pick an experiment that resolves the unknown. If this repeats, pass explicit backend/fleet capability context from the server into the agent prompt. | fleet `nodes`/`idle_duration`, current idle/running instances, backend type evidence, agent reasoning line, chosen run type, whether task/dev was skipped | the next live run still misclassifies JarvisLabs/Lambda/Verda/Nebius/SSH/Kubernetes capacity; then make backend capability context server-generated instead of prompt-inferred | +| Task/dev probe is too shallow | Treat `nvidia-smi` and driver strings as host evidence only. A useful probe for service promotion should exercise the selected image or install path, Python/framework runtime, model/auth/cache access when feasible, server start on the intended port, and a local health or model API request if possible. If the probe exits before those checks, classify it as failed/inconclusive and either fix the probe or continue experimenting before promotion. | task/dev config, logs, driver evidence, framework import/version, model download result, local server/probe result, final service verification | two runs over-interpret partial probes; then require a structured pre-promotion artifact before the agent may submit the final service | +| Agent assumes global offers are usable | Enforce at the CLI boundary, not only in prompt text. The workspace-local `dstack` shim blocks fleet applies and blocks `dstack offer` / run applies that omit allowed fleets or violate accepted endpoint profile constraints. | fleet list/get output, fleet-filtered offers, backend/hardware reasoning, whether submitted configs carry the allowed fleet/profile constraints, cleanup, CLI guard block output | agent still finds a path around the guard, creates/edits a fleet, or submits a run outside allowed constraints | +| Host driver/CUDA compatibility is invisible in offers | Treat driver/runtime compatibility as uncertainty. Some backends vary host drivers by provider, GPU pool, region, or node, and dstack fleets/offers may not expose it. The agent should infer risk from provider/GPU/region evidence when possible and verify with `nvidia-smi`, run JSON/logs, dev environments, or one-shot tasks before trusting unpinned package installs. | `nvidia-smi`/driver evidence, package/image CUDA requirements, provider/GPU/region evidence, failed log root cause | driver mismatch recurs on the same provider; then bias recipes toward known-compatible images/package pins or avoid that offer class | +| Agent logs are noisy or unhelpful | Split logs into two layers: endpoint log gets natural-language decision updates authored by the agent; workspace trace stores command output, YAML, raw logs, and final report. Do not impose labels/categories/templates, and do not put full CLI output or service logs in endpoint logs. | `dstack endpoint logs ` readability, workspace artifacts, ability to diagnose failure without server stdout | endpoint logs still duplicate/replay, hide important state, force reading server logs for normal debugging, or stay too sparse to follow the investigation | +| Restart during `prototyping` duplicates work or loses the final run | Same endpoint configuration lifecycle reuses the same agent session/workspace; dead same-host Claude processes are relaunched in that workspace; new endpoint lifecycle creates the next session and passes older session summaries as read-only context. Still reconcile before launch: process metadata, final report, endpoint submission rows, linked `service_run_id`, and live submitted runs. | restart test with a live process, restart test after final report before link, endpoint submissions, linked run, no duplicate paid process | reconciliation logic becomes fragile, or multi-host restarts cannot observe enough process/workspace state; then consider moving agent execution to a separately tracked durable task model | +| Stop during `prototyping` leaks resources | Add process cancellation and submitted-run cleanup using `EndpointRunSubmissionModel`, not name heuristics. Endpoint stop should request abort, stop linked/submitted non-terminal runs, then finish through the normal run lifecycle. | stop during Claude process, stop after a submitted run appears, final run statuses, endpoint stopped, no running GPU run | cancellation depends on runtime behavior we cannot rely on; then isolate agent execution in a managed process/task with explicit supervisor state | +| Agent cost governance is not durable yet | Keep budget/cost accounting out of the v1 public endpoint schema. Add it later only when the runtime exposes reliable usage and dstack persists spend per endpoint agent session before any resume/retry starts another process. | Claude reported usage/cost, persisted session spend, refusal path when exhausted | Claude Code cannot expose reliable usage data; then use a stricter wall-clock/process cap plus operator-visible warning until runtime changes | +| Multiple learned recipes accumulate | Keep recipe selection simple: use the first stored recipe that has available offers under the endpoint's effective fleets/profile constraints. When the agent saves a newly verified recipe, store it before older recipes so immediate reuse sees the latest proof. Do not delete or rewrite older recipes until the harness can prove they are not reproducible under their claimed conditions. | recipe list by model, service resources, validation evidence, reuse success/failure per recipe, whether the freshly saved recipe is selected by a no-spend preview | users need control over a specific recipe; then add an explicit later `recipe: ` endpoint option before adding automatic ranking or update policy | | Agent harness design overfits early Qwen runs | Maintain a scenario ladder: one tiny public model, one slightly larger common model, one gated/HF-token model, one no-offers/constraint case, one failure-cleanup case. Change harness only after comparing patterns across at least two scenarios unless the bug is clearly blocking. | per-scenario run notes, failure categories, command counts, spend, preset reuse result | the first two scenarios expose contradictory needs; then pause and rework the harness contract rather than patching locally | Near-term implementation strategy: 1. Prefer prompt/context and artifact requirements for the first two real runs. They are cheaper to change and reveal what the agent naturally does. -2. Add server-side validation only where the failure would be expensive or dangerous: wrong project/user run, missing service `model`, missing final verification, budget exhaustion, leaked non-terminal runs, invalid preset shape. +2. Add server-side validation only where the failure would be expensive or dangerous: wrong project/user run, missing service `model`, missing final verification, leaked non-terminal runs, invalid preset shape. 3. Keep every hardening change tied to a reproducible trace: command transcript, endpoint log, final report, saved preset, and dstack run state. 4. After each real run, classify the issue as one of: dstack/backend provisioning bug, agent decision bug, prompt/context gap, preset contract bug, lifecycle/recovery bug, or UX/logging bug. Fix the right layer; do not let agent failures hide dstack/backend failures. 5. Be willing to change runtime approach if Claude Code cannot reliably run non-interactively, expose usage, cancel, preserve artifacts, or package cleanly in server installs/Docker. @@ -120,16 +187,18 @@ Near-term implementation strategy: ### Adjusted from the original plan - Preset service responsibility was narrowed to **storage only**. Matching, planning, and service apply live outside `EndpointPresetService`. -- Presets now store `replica_spec_groups` as ordered groups of `ResourcesSpec`, separate from the service config. Group order must match `ServiceConfiguration.replica_groups`; the implicit no-replica-groups case uses group `"0"`. -- Presets do **not** store backend, region, or instance type in v1. They store the tested resource topology that can be replanned against current fleets. -- A learned preset records how many registered running replicas existed per replica group at save time. Autoscaling metrics, scaling validation, and benchmark metadata are Later. +- Presets are now model-level files with one or more `recipes`. The recipe `service` is the scheduling source of truth; `validations` store exact running replica resources in service replica-group order. +- New learned GPU recipes should be vendor-aware because serving recipes are not portable across accelerator vendors. Existing vendorless local recipes are preserved on read; the agent harness is responsible for authoring vendor-aware final service YAML, and the learned-preset builder may fill the vendor from exact validation hardware before saving. +- Presets do **not** store backend, region, or instance type in v1. They store service resource requirements plus exact validation evidence that can be replanned against current fleets. +- A learned recipe records how many registered running replicas existed per replica group at save time through `validations[*].replicas[*].resources`. Autoscaling metrics, scaling validation, and benchmark metadata are Later. - Preset matching does **not** force `creation_policy: reuse`; default `reuse-or-create` may use elastic fleets that can provision new instances. - Endpoint plan `preset_policy` remains the configured/default policy; the selected path (`preset`, `agent`, or `none`) is represented only by `provisioning_plan`. - Endpoint config changes are handled like current run UX: prompt to stop/override, with the backing service cleanup performed server-side. In-place update/rolling redeploy is Later. - Later endpoint config updates must reuse the existing service-run `get_plan`/`apply_plan`/rolling deployment machinery, and endpoint DB updates must be guarded by a configuration/deployment version for multi-server safety. +- Endpoint-level `resources` are not part of v1. V1 placement constraints are the accepted `ProfileParams` plus the preset/service resources. If endpoint `resources` are added later, they must be a hard constraint with clear merge semantics for single-service and replica-group services; they must not be a prompt-only hint. - Stop/override applies only to non-terminal existing endpoints. Terminal endpoints are replaceable like finished runs. - Serving-run-name lookup is a conflict check for non-terminal runs only. Terminal conflicting runs follow normal run submission semantics: the run apply path can delete/recreate them, while endpoint code must not adopt/delete unrelated runs by name. -- No generic server-side endpoint health probe in v1. Existing service probes/registration are the server readiness source. In the agent path, the agent itself must perform final functional verification before reporting a final candidate. +- No generic server-side endpoint health probe in v1. Existing service probes/registration are the server readiness source. In the agent path, the agent itself must perform final functional verification before reporting a final service. - Automatic provisioning retry is Later. A failed endpoint is terminal for v1. - Frontend UI is explicitly Later; no UI is required for v1. - `DSTACK_AGENT_ANTHROPIC_API_KEY` remains the official server-agent env var. A local `DSTACK_ANTHROPIC_API_KEY` must be mapped/exported to that name if used. @@ -157,7 +226,7 @@ dstack apply -f endpoint.dstack.yml ├─ preset matches existing fleets? → submit service run from preset ├─ else, agent enabled (DSTACK_AGENT_ANTHROPIC_API_KEY)? → agent deploys a service └─ else → FAILED ("no matching preset found and server agent disabled") - └─ endpoint: CLAUDING (agent investigates/deploys, for no-preset agent path) + └─ endpoint: PROTOTYPING (agent investigates/deploys, for the create-recipe path) └─ endpoint: PROVISIONING (service run starting, model pulling, service probes passing) └─ endpoint: RUNNING (service has a registered running job and model URL) ``` @@ -167,10 +236,10 @@ The endpoint's deliverable is an OpenAI-compatible base URL (from the backing se ### v1 scope (this plan) - New configuration type + DB model + REST API + CLI (`dstack apply`, `dstack endpoint list|get|logs|stop|preset`). -- `EndpointPipeline` background processing: SUBMITTED → CLAUDING/PROVISIONING → RUNNING/FAILED, STOPPING → STOPPED, crash recovery, and ownership-safe reconciliation. +- `EndpointPipeline` background processing: SUBMITTED → PROTOTYPING/PROVISIONING → RUNNING/FAILED, STOPPING → STOPPED, crash recovery, and ownership-safe reconciliation. - Preset subsystem: `EndpointPresetService` interface + local-directory implementation for storing/loading presets; endpoint planning code separately matches loaded presets against existing fleets, including elastic fleets that can provision new instances, via the run planner; successful agent deployments are saved back as sanitized local presets. -- Agent subsystem: `AgentService` interface + real Claude agent runtime implementation, a CLI-first execution workspace that lets the agent use the real `dstack` binary, vendored prompt/context, recipe/hardware grounding, and a minimal structured handoff for the final candidate. Raw LLM API loops are not accepted for v1. -- Endpoint readiness follows the backing dstack service lifecycle on the server side: a service run is usable once it is RUNNING, has a registered running job, and exposes a model URL. The agent path additionally requires the agent to verify the final candidate with a model request before reporting success; the server does not add a generic duplicate endpoint probe in v1. +- Agent subsystem: `AgentService` interface + real Claude agent runtime implementation, a CLI-first execution workspace that lets the agent use the real `dstack` binary, vendored prompt/context, recipe/hardware grounding, and a minimal structured handoff for the final service. Raw LLM API loops are not accepted for v1. +- Endpoint readiness follows the backing dstack service lifecycle on the server side: a service run is usable once it is RUNNING, has a registered running job, and exposes a model URL. The agent path additionally requires the agent to verify the final service with a model request before reporting success; the server does not add a generic duplicate endpoint probe in v1. - `DSTACK_AGENT_ANTHROPIC_API_KEY` + related settings; real agent runtime dependencies installed automatically by normal server install and Docker deployment paths. - Endpoint apply/log UX: `dstack apply -f endpoint.dstack.yml` shows an advisory plan and confirmation first, then follows endpoint progress by polling server-side status/log storage; `dstack endpoint logs ` shows endpoint agent/progress logs, while backing service logs remain available via top-level `dstack logs `. No frontend UI in v1. @@ -207,7 +276,7 @@ New file `src/dstack/_internal/core/models/endpoints.py`: class EndpointStatus(str, Enum): SUBMITTED = "submitted" PROVISIONING = "provisioning" - CLAUDING = "clauding" # server agent is investigating/deploying + PROTOTYPING = "prototyping" # server agent is investigating/deploying RUNNING = "running" # backing service is ready STOPPING = "stopping" STOPPED = "stopped" @@ -301,7 +370,7 @@ class EndpointModel(PipelineModelMixin, BaseModel): - Endpoint configuration includes `preset_policy` (`reuse`, `create`, `reuse-or-create`, default `reuse-or-create`). This is distinct from profile `creation_policy`: `preset_policy` chooses whether endpoint provisioning may reuse a tested service recipe or ask the server agent to create one; `creation_policy` still controls whether the resulting service can reuse existing instances or provision new ones from elastic fleets. - New core model `EndpointPlan`: `project_name`, `user`, `configuration`, `configuration_path`, `current_resource`, `action`, `preset_policy`, and `provisioning_plan`. `preset_policy` is the configured/default policy shown in the CLI; the selected path (`preset`, `agent`, or `none`) belongs in `provisioning_plan`. - `provisioning_plan` is a small discriminated union: - - `type="preset"`: a **provisionable preset** was selected. This includes `preset_name`, `service_name`, `replica_spec_groups: list[EndpointPlanReplicaSpecGroup]`, and `job_offers: list[EndpointPlanJobOffers]` derived from `runs.get_plan` (enough for the CLI to print the selected preset, stable run-like scheduling properties, and first matching offers, without dumping the full run plan). + - `type="preset"`: a **provisionable preset recipe** was selected. This includes `preset_model`, `recipe_id`, `service_name`, and `job_offers: list[EndpointPlanJobOffers]` derived from `runs.get_plan` (enough for the CLI to print the selected preset/recipe, stable run-like scheduling properties, and first matching offers, without dumping the full run plan). - `type="agent"`: no provisionable preset was found and the policy allows creation. The plan may include a short reason such as "matching preset qwen has no available offers"; the initial plan must not invent final offers because the agent may explore hardware and service shapes within endpoint/profile constraints. - `type="none"`: no provisionable path is available, with the exact failure message that the pipeline will use (for example, `preset_policy: reuse` with no matching preset or only no-offer presets, or `preset_policy: reuse-or-create` with no provisionable preset and no usable server agent). - Preset terminology: @@ -318,7 +387,7 @@ class EndpointModel(PipelineModelMixin, BaseModel): - `src/dstack/_internal/cli/services/configurators/endpoint.py`: `EndpointConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator)` with `TYPE = ApplyConfigurationType.ENDPOINT`. Model on `VolumeConfigurator`: - `apply_configuration`: resolve env sentinels (`apply_env_vars`), call `api.client.endpoints.get_plan(...)`, print the endpoint plan, confirm, then `api.client.endpoints.create(...)`; on name collision: if the existing endpoint has the same config, print no changes and exit unless `--force`; if the config changed, show that the endpoint and its backing service will be stopped and replaced, then require confirmation (or `-y`) before calling `endpoints.stop`, polling until the endpoint becomes terminal, and creating a fresh endpoint. The CLI must not stop backing runs directly. This mirrors run apply's "stop and override" v1 behavior, not an in-place update; in-place update/rolling redeploy is Later (§12). - - Plan output: keep it close to run apply. Print one stable properties table: project, user, configuration path, type, spot policy, max price, preset policy, and preset name only when a provisionable preset was selected. Do not show model, action, endpoint name as a separate row, resources as a separate property, backing service name, agent internals, or a separate "Provisioning" section. If a provisionable preset matched, print offers underneath using the run-style offers table; that offers table is where the concrete resource information belongs. If only no-offer presets matched, print one short message below the table; with `reuse-or-create` and an available agent, continue to the agent path. + - Plan output: keep it close to run apply. Print one stable properties table: project, user, configuration path, type, spot policy, max price, preset policy, and preset model/recipe only when a provisionable recipe was selected. Do not show model, action, endpoint name as a separate row, resources as a separate property, backing service name, agent internals, or a separate "Provisioning" section. If a provisionable recipe matched, print offers underneath using the run-style offers table; that offers table is where the concrete resource information belongs. If only no-offer recipes matched, print one short message below the table; with `reuse-or-create` and an available agent, continue to the agent path. - TODO: For `preset_policy: create` or agent fallback, `dstack apply` may later show initial candidate offers the agent can consider. These offers are advisory only: they must not be presented as selected endpoint resources or final hardware because the agent may still choose a different working service shape after investigation. - Non-detached apply should **imitate attached progress without using attach**: poll `endpoints.get` for status and stream the endpoint's own progress log stream. Do not reserve local ports, open SSH tunnels, or call `run.attach`. Backing service stdout/stderr remains available through `dstack logs `. - Detached apply (`-d`) submits and exits with the normal `submitted, detaching...` message. Follow-up commands (`dstack endpoint list`, `dstack endpoint get --json`, and `dstack endpoint logs `) are documented/help text, not extra foreground hints. @@ -348,7 +417,7 @@ New file `src/dstack/_internal/server/background/pipeline_tasks/endpoints.py`, c - `EndpointPipelineItem(PipelineItem)` + `{status}`; `EndpointPipeline(Pipeline[EndpointPipelineItem])`, `hint_fetch_model_name -> EndpointModel.__name__`. - Tuning: `workers_num=4` (each in-flight agent run occupies a worker for minutes — this is the per-replica concurrency bound for paid agent sessions), `min_processing_interval=10s`, `lock_timeout=30s`, `heartbeat_trigger=15s`. The `Heartbeater` extends the lease indefinitely during a long `process()` call (precedent: `instances/cloud_provisioning.py`), so minutes-long agent steps are safe while the replica lives. - `EndpointFetcher.fetch` — copy the volume fetch query shape, with the endpoint status filter and a longer effective interval for RUNNING rows (precedent: `InstanceFetcher`, `instances/__init__.py:190-211`, uses `min_processing_interval * 2` for steady-state statuses; we use a larger multiplier via an explicit condition): - - `WHERE status IN (SUBMITTED, CLAUDING, PROVISIONING, STOPPING) OR (status == RUNNING AND last_processed_at <= now - ENDPOINT_RUNNING_CHECK_INTERVAL)` + - `WHERE status IN (SUBMITTED, PROTOTYPING, PROVISIONING, STOPPING) OR (status == RUNNING AND last_processed_at <= now - ENDPOINT_RUNNING_CHECK_INTERVAL)` - `AND (last_processed_at <= now - min_processing_interval OR last_processed_at == created_at)` - `AND (lock_expires_at IS NULL OR lock_expires_at < now)` - `AND (lock_owner IS NULL OR lock_owner == EndpointPipeline.__name__)` @@ -365,13 +434,13 @@ New file `src/dstack/_internal/server/background/pipeline_tasks/endpoints.py`, c ``` SUBMITTED ──(preset selected; deterministic service run name is taken)──► FAILED "run name '' is taken by an existing run" SUBMITTED ──(preset matched; run submitted)───────► PROVISIONING (method=preset:) -SUBMITTED ──(no preset, agent on)─────────────────► CLAUDING (method=agent) +SUBMITTED ──(no preset, agent on)─────────────────► PROTOTYPING (method=agent) SUBMITTED ──(preset_policy=reuse; no preset)──────► FAILED "No matching endpoint presets found." SUBMITTED ──(effective policy=create; agent off)──► FAILED "No matching endpoint presets found. Preset policy create requires the server agent, but DSTACK_AGENT_ANTHROPIC_API_KEY is not set." -CLAUDING ─(agent reports verified service run; run not ready yet)──► CLAUDING (service_run_id linked) -CLAUDING ─(linked run RUNNING + replica registered + model URL)────► RUNNING +PROTOTYPING ─(agent reports verified service run; run not ready yet)──► PROTOTYPING (service_run_id linked) +PROTOTYPING ─(linked run RUNNING + replica registered + model URL)────► RUNNING PROVISIONING ─(preset run RUNNING + replica registered + model URL)► RUNNING PROVISIONING ─(run finished/soft-deleted)──────────────────────────► FAILED RUNNING ─(run gone/failed/deleted)─────────────────────────────────► FAILED (stop backing run) # later: agent retry policy @@ -382,8 +451,8 @@ V1 does **not** automatically retry failed provisioning attempts. A backing run #### `_process_submitted_endpoint` 1. Parse `EndpointConfiguration`; ask endpoint planning code (`planning.py::find_matching_preset_plan(...)`) to list presets through `EndpointPresetService`, build candidate run specs, and call the run planner (§7). -2. If a provisionable preset matches: check the run name in that selected plan. A non-terminal run conflict that is not the linked `service_run_id` fails with a clear message; terminal conflicts are left to normal run submission, which can recycle them like service apply. Then submit the matching plan in a fresh session as the endpoint's creator user through a small internal helper that mirrors the run router (`refresh_ssh_key` if needed, `get_plan`, then `apply_plan`/`submit_run` with the same policy-applied spec), and return `{status: PROVISIONING, provisioning_method: f"preset:{preset.name}", service_run_id: run.id}`. `register_service` runs synchronously during submission and can raise (`FORBID_SERVICES_WITHOUT_GATEWAY`, referenced gateway missing) — catch `ServerClientError` ⇒ FAILED with the message. -3. Else if agent enabled (`settings.AGENT_ANTHROPIC_API_KEY` set and the real agent runtime is available): return `{status: CLAUDING, provisioning_method: "agent"}` — the agent runs in the CLAUDING handler (keeps SUBMITTED processing fast and makes long agent work visible in CLI/API). Do not block this path on the preset path's deterministic service-run-name convention; the agent may use its own concise candidate/final run names, and the server links the final run by ID after validating project/user/config. +2. If a provisionable preset recipe matches: check the run name in that selected plan. A non-terminal run conflict that is not the linked `service_run_id` fails with a clear message; terminal conflicts are left to normal run submission, which can recycle them like service apply. Then submit the matching plan in a fresh session as the endpoint's creator user through a small internal helper that mirrors the run router (`refresh_ssh_key` if needed, `get_plan`, then `apply_plan`/`submit_run` with the same policy-applied spec), and return `{status: PROVISIONING, provisioning_method: f"preset:{preset.model}#{recipe.id}", service_run_id: run.id}`. `register_service` runs synchronously during submission and can raise (`FORBID_SERVICES_WITHOUT_GATEWAY`, referenced gateway missing) — catch `ServerClientError` ⇒ FAILED with the message. +3. Else if agent enabled (`settings.AGENT_ANTHROPIC_API_KEY` set and the real agent runtime is available): return `{status: PROTOTYPING, provisioning_method: "agent"}` — the agent runs in the PROTOTYPING handler (keeps SUBMITTED processing fast and makes long agent work visible in CLI/API). Do not block this path on the preset path's deterministic service-run-name convention; the agent uses strict numbered submission names, and the server links the final run by ID after validating project/user/config. 4. Else FAILED (message above; if the key is set but the runtime is unavailable, say the server agent implementation/runtime is unavailable; normal server installs and Docker images must include runtime dependencies once the implementation lands). #### `_process_provisioning_endpoint` — reconcile-first @@ -392,16 +461,16 @@ V1 does **not** automatically retry failed provisioning attempts. A backing run - run in `finished_statuses()` or soft-deleted → FAILED with the run error/status. Automatic retry is Later. - run `TERMINATING` → no-op (wait). - run still starting (SUBMITTED/PROVISIONING/PULLING) → no-op; stay PROVISIONING. -3. **CLAUDING / no run yet + method=agent** — the long step: - a. Needed hardening: before launching or relaunching the agent, reconcile the endpoint workspace, existing final-report artifacts, endpoint submission rows, and live candidate/final runs. Without this, a server restart can start a fresh agent process for an endpoint whose previous agent process had already submitted work but had not linked `service_run_id` yet. Cleanup of endpoint-submitted runs must use explicit `EndpointRunSubmissionModel` rows, never name matching. - b. Run `AgentService.provision_endpoint(...)` (§8) inside the worker — heartbeater keeps the lease alive while the agent subprocess runs. V0 still needs stop-time cancellation and restart-safe recovery: stop can mark the endpoint `STOPPING`, but the worker must learn to stop/abort the live agent process instead of waiting for it to exit naturally. Do not add a generic endpoint provisioning timeout here; real agent-session budgets belong inside the harness once token/cost/spend semantics are defined. - c. On structured success `{run_id}`/linked service run → set `service_run_id` and keep status `CLAUDING` until the linked service run also satisfies normal dstack service readiness. Once ready, save the learned preset and mark the endpoint `RUNNING`. On failure/abort → FAILED with the agent's summary in `status_message` + stop only the linked service run; cleanup of non-final candidate runs must use explicit endpoint submission records, not name matching. +3. **PROTOTYPING / no run yet + method=agent** — the long step: + a. Before launching or relaunching the agent, reconcile the endpoint workspace, existing final-report artifacts, endpoint submission rows, and live submitted/final runs. Same endpoint configuration lifecycle reuses the same agent session/workspace; if the same-host Claude process is gone without a terminal report, the server relaunches Claude in that workspace so it can resume from artifacts and submitted-run evidence. A new endpoint lifecycle starts the next session and gives the agent `previous_sessions.md` as read-only context. Cleanup of endpoint-submitted runs must use explicit `EndpointRunSubmissionModel` rows, never name matching. + b. Run `AgentService.provision_endpoint(...)` (§8) inside the worker — heartbeater keeps the lease alive while the agent subprocess runs. Stop-time cancellation exists for same-host Claude processes; multi-host restart/supervision remains a hardening item. Do not add a generic endpoint provisioning timeout here; real agent-session budgets belong inside the harness once token/cost/spend semantics are defined. + c. On structured success `{run_id}`/linked service run → set `service_run_id` and keep status `PROTOTYPING` until the linked service run also satisfies normal dstack service readiness. Once ready, save the learned preset and mark the endpoint `RUNNING`. On failure/abort → FAILED with the agent's summary in `status_message` + stop only the linked service run; cleanup of non-final submitted runs must use explicit endpoint submission records, not name matching. 4. **No run yet + method=preset**: reachable only if the preset run vanished before the endpoint recorded its linked service run — FAILED. Automatic retry is Later. #### `_process_running_endpoint` Runs at the slower RUNNING cadence (`ENDPOINT_RUNNING_CHECK_INTERVAL`, default 60s): 1. Run liveness: backing run missing/soft-deleted/`finished_statuses()`/`TERMINATING` ⇒ FAILED "Backing service run is " (v1; re-provisioning is a Later "agent retry policy" item). This also catches out-of-band `dstack stop`/`delete` of the run by users. -2. No generic server-side endpoint model probe in v1. The backing service owns probes and registration for lifecycle readiness. In the agent path, the agent must make a final model request and include the result in its final report before the server links the candidate. +2. No generic server-side endpoint model probe in v1. The backing service owns probes and registration for lifecycle readiness. In the agent path, the agent must make a final model request and include the result in its final report before the server links the final service. **Rule: every transition to FAILED issues a one-shot `stop_runs(abort=False)` for a still non-terminal backing run** (RunPipeline completes the termination asynchronously; no waiting needed since FAILED endpoints aren't re-fetched). Stopping a FAILED endpoint later is a no-op unless a linked run still needs cleanup. @@ -410,7 +479,7 @@ Two-phase, server-side, and intentionally simple in v1: (1) backing run present ### 6.3 Crash recovery summary -Replica dies mid-step ⇒ heartbeats stop ⇒ lease expires (≤ ~30s) ⇒ another replica re-fetches the row. V1 handlers are idempotent around the linked `service_run_id`; preset-run-name conflicts are treated as conflicts, not ownership. `EndpointRunSubmissionModel` preserves endpoint-submitted run history so later agent attempts can recover without relying on names. Agent session budgeting is Later and should be explicit to the agent harness, not a generic endpoint provisioning timeout. Automatic retry is Later. +Replica dies mid-step ⇒ heartbeats stop ⇒ lease expires (≤ ~30s) ⇒ another replica re-fetches the row. V1 handlers are idempotent around the linked `service_run_id`; preset-run-name conflicts are treated as conflicts, not ownership. `EndpointAgentSessionModel` tracks the current Claude process/workspace for an endpoint configuration lifecycle. `EndpointRunSubmissionModel` preserves dstack runs created by that session so recovery does not rely on fragile name heuristics. Agent session budgeting is Later and should be explicit to the agent harness, not a generic endpoint provisioning timeout. Automatic retry is Later. ### 6.4 Endpoint-level model probes (deferred) @@ -420,7 +489,7 @@ Later, server-side endpoint retry/hardening may add its own explicit verificatio ### 6.5 Run identity & linkage (decision) -- **Backing service run name**: the preset path uses `get_endpoint_serving_run_name(endpoint.name)` (`-serving` when it fits, otherwise the endpoint name) because the server submits that service itself. The agent path may use any concise, unique run name. In both paths, `EndpointModel.service_run_id` is the authoritative link for readiness, URL derivation, logs, and v1 cleanup. Name lookup is not an ownership proof and must not be used to adopt or destroy a run. +- **Backing service run name**: the preset path uses `get_endpoint_serving_run_name(endpoint.name)` (`-serving` when it fits, otherwise the endpoint name) because the server submits that service itself. The agent path uses numbered endpoint submissions: `-1`, `-2`, and so on. In both paths, `EndpointModel.service_run_id` is the authoritative link for readiness, URL derivation, logs, and v1 cleanup. Name lookup is not an ownership proof and must not be used to adopt or destroy a run. - **Endpoint-created run ownership:** `EndpointModel.service_run_id` points to the current/latest service run. `EndpointRunSubmissionModel` records ordered endpoint-submitted run history. Keep `RunModel` endpoint-agnostic. - Consequences for conservative v1: a non-terminal preset-path service run name not linked by `service_run_id` ⇒ endpoint FAILED with a clear message before preset submission. A terminal conflicting run is handled by the existing run submission path. Agent retry policy and cleanup of non-final agent experiment runs are Later/Path B work. @@ -443,11 +512,12 @@ Constraints: Rules: - `EndpointModel.service_run_id` remains the single current serving run. -- `EndpointRunSubmissionModel` is history/provenance for endpoint-submitted runs, not the serving-run selector. +- `EndpointRunSubmissionModel` is endpoint-submitted run history, not the serving-run selector. - Preset-path run-name lookup remains conflict detection for non-terminal runs only. CLI/user confirmation may stop that conflicting run, but the background worker must not auto-adopt or auto-delete it by name. -Implementation note: -- `runs.apply_plan()` can commit internally, so endpoint code records the submission immediately after a run is accepted and before moving on. If the agent creates multiple prototype/candidate runs inside one call, its submit tool must record each `EndpointRunSubmissionModel` row before the next submission. +Implementation notes: +- `runs.apply_plan()` can commit internally, so preset-path endpoint code records the submission immediately after a run is accepted and before moving on. +- In the agent path, Claude records every submitted dev environment, task, or service in `submissions.jsonl`. The server reconciles strict numbered submitted run names and reported run IDs into `EndpointRunSubmissionModel` rows; non-strict names are ignored for ownership and cleanup. --- @@ -458,125 +528,138 @@ Preset matching and run-plan construction live in `src/dstack/_internal/server/s ### 7.1 What a preset is -A preset is **one tested endpoint deployment topology on one tested set of replica spec groups**. It is not a generic service recipe that dstack is free to reshape onto arbitrary GPUs, and it is not a bundle of alternative hardware variants. If the same model is tested on L4 and A10G, that is two presets. If one tested service uses multiple replicas or multiple replica groups, that is still one preset. +A preset is a **model-level set of tested deployment recipes**. It is not a single generated name and it is not only a service YAML. The model is the lookup key; each recipe has an internal `id`. In v1 the local file is a small wrapper around: - `model`: the endpoint model this preset satisfies and the matching key; -- `service`: the serving recipe and service shape (image, commands, port, model mapping, env names, volumes, probes, scaling, replica count/range, or replica groups); -- `replica_spec_groups`: ordered replica groups with one scheduling `resources` value and one `tested_resources` entry per verified running replica. +- `recipes`: one or more tested deployment recipes for the model; +- `recipes[*].service`: the serving recipe, service shape, and scheduling resources used for reuse/offer matching; +- `recipes[*].validations`: exact verified hardware evidence from successful deployments, ordered to match `service.replica_groups`. -The wrapper is deliberate: scheduling requirements and placement evidence are separated from the serving recipe in the preset artifact, then compiled into a normal `ServiceConfiguration` before calling the existing run planner. This keeps the stored preset clear without introducing a new provisioning path. +The wrapper is deliberate: a recipe is compiled directly into a normal `ServiceConfiguration` before calling the existing run planner. There is no new scheduler path. The service section owns resources because that is already how dstack services describe placement; validations are evidence, not scheduling input. -Homogeneous service example (`qwen3-32b-vllm-h100x4.dstack.yml`): no explicit `service.replicas` list means there is one implicit replica group (`"0"`). The loader derives the service's top-level `resources` from the group's `resources` before planning/submission. The group's `tested_resources` records where the replicas actually ran when verified. +Homogeneous service example (`qwen3-32b-vllm-h100x4.dstack.yml`): no explicit `service.replicas` list means there is one implicit service replica group. `validations[0].replicas[0].resources` records the exact resources of each running replica observed when the recipe was verified. ```yaml type: endpoint-preset model: Qwen/Qwen3-32B -service: - image: vllm/vllm-openai:latest - commands: - - | - vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 \ - --tensor-parallel-size $DSTACK_GPUS_NUM - port: 8000 - model: Qwen/Qwen3-32B - env: - - HF_TOKEN - volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true -replica_spec_groups: - - name: "0" # optional for the implicit group; stored explicitly when saving - resources: - shm_size: 16GB - gpu: H100:4 - tested_resources: - - cpu: 64 - memory: 512GB - disk: 200GB - gpu: H100:80GB:4 +recipes: + - id: 8f3a12c4 + service: + image: vllm/vllm-openai:latest + commands: + - | + vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 \ + --tensor-parallel-size $DSTACK_GPUS_NUM + port: 8000 + model: Qwen/Qwen3-32B + env: + - HF_TOKEN + volumes: + - instance_path: /root/.cache + path: /root/.cache + optional: true + resources: + shm_size: 16GB + gpu: nvidia:H100:4 + validations: + - replicas: + - resources: + - cpu: 64 + memory: 512GB + disk: 200GB + gpu: nvidia:H100:80GB:4 ``` -Replica-group example: `replica_spec_groups` is sorted in exactly the same order as `service.replicas`, and every group name must match. The loader derives each replica group's service `resources` from the corresponding preset group's `resources`. `tested_resources` records the exact instances that were running when the preset was verified. This supports an existing dstack service shape with different resource specs per group without making v1 responsible for inventing advanced serving architectures. +Replica-group example: `validations[*].replicas` is sorted in exactly the same order as `service.replica_groups`. There is no separate group name because service replica-group order is already explicit. This supports existing dstack services with different resource specs per group without making v1 responsible for inventing advanced serving architectures. ```yaml type: endpoint-preset model: Qwen/Qwen3-32B -service: - port: 8000 - model: Qwen/Qwen3-32B - replicas: - - name: router - count: 1 - image: ghcr.io/example/router:latest - commands: - - python router.py - - name: worker - count: 2 - image: vllm/vllm-openai:latest - commands: - - vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 -replica_spec_groups: - - name: router - resources: - cpu: 4 - tested_resources: - - cpu: 8 - memory: 16GB - disk: 100GB - gpu: 0 - - name: worker - resources: - shm_size: 16GB - gpu: H100:4 - tested_resources: - - cpu: 64 - memory: 512GB - disk: 200GB - gpu: H100:80GB:4 - - cpu: 64 - memory: 512GB - disk: 200GB - gpu: H100:80GB:4 +recipes: + - id: 4c38b901 + service: + port: 8000 + model: Qwen/Qwen3-32B + replicas: + - name: router + count: 1 + image: ghcr.io/example/router:latest + commands: + - python router.py + resources: + cpu: 4 + - name: worker + count: 2 + image: vllm/vllm-openai:latest + commands: + - vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 + resources: + shm_size: 16GB + gpu: nvidia:H100:4 + validations: + - replicas: + - resources: + - cpu: 8 + memory: 16GB + disk: 100GB + gpu: 0 + - resources: + - cpu: 64 + memory: 512GB + disk: 200GB + gpu: nvidia:H100:80GB:4 + - cpu: 64 + memory: 512GB + disk: 200GB + gpu: nvidia:H100:80GB:4 ``` V1 constraints: -- `service` must not contain `name`, `ProfileParams`, top-level `resources`, or replica-group `resources`. Endpoint `ProfileParams` constrain placement/fleets/pricing; they do not change the preset's tested replica spec groups. -- `replica_spec_groups` order is part of the contract. If `service.replicas` is a list, `replica_spec_groups[i].name == service.replicas[i].name` for every group. If there is no replica-group list, the preset has exactly one implicit group (`"0"`), analogous to the default service replica group. -- Each replica group has one `resources` value used for offer matching and service submission. `tested_resources` may contain multiple exact entries, one per verified running replica. If the service needs different scheduling requirements, model those as separate replica groups. -- A preset has exactly one tested replica-spec topology. Hardware alternatives, benchmarked variants, and curated multi-choice registries are Later. +- `service` must not contain `name` or `ProfileParams`. Endpoint `ProfileParams` constrain placement/fleets/pricing on top of the recipe service. +- `service` must contain resources either at the top level or on every explicit replica group. Those resources are the scheduling contract used by reuse planning. +- New learned GPU service resources should specify a vendor. The agent harness should write a broad vendor-aware requirement such as `gpu: nvidia:16GB..24GB:1` after it chooses or verifies NVIDIA hardware. The loader preserves existing vendorless local recipes, but explicit vendor conflicts with validation evidence are invalid. +- `validations[*].replicas` order is part of the contract. If `service.replicas` is a list, `validations[*].replicas[i]` corresponds to `service.replicas[i]`. If there is no replica-group list, each validation has exactly one implicit replica group. +- Each validation replica group contains `resources: list[ResourcesSpec]`, one exact entry per verified running replica/instance in that group. +- Hardware alternatives, benchmarked variants, automatic recipe updates, and metrics on validations are Later. ### 7.2 Interfaces ```python class EndpointPreset(CoreModel): - name: str # file stem model: str # matching key - replica_spec_groups: list[EndpointPresetReplicaSpecGroup] # ordered like service replica groups - """Sorted to match `configuration.replica_groups`; group "0" is the implicit group.""" - configuration: ServiceConfiguration # compiled service config, not the raw file + recipes: list[EndpointPresetRecipe] -class EndpointPresetReplicaSpecGroup(CoreModel): - """Ordered to match `ServiceConfiguration.replica_groups`; "0" is the implicit group.""" - name: str # replica group name; "0" for implicit group - resources: ResourcesSpec # scheduling requirements for each replica in the group - tested_resources: list[ResourcesSpec] # exact verified resources for running replicas +class EndpointPresetRecipe(CoreModel): + id: str # hash of normalized service recipe in v1 + service: ServiceConfiguration # compiled service config without name/profile fields + validations: list[EndpointPresetValidation] + +class EndpointPresetValidation(CoreModel): + replicas: list[EndpointPresetValidationReplica] + """Ordered to match `ServiceConfiguration.replica_groups`.""" + +class EndpointPresetValidationReplica(CoreModel): + resources: list[ResourcesSpec] # exact resources for running replicas in this group class EndpointPresetService(ABC): # storage abstraction only — local dir first, S3/git later @abstractmethod async def list_presets(self, project_name: str) -> list[EndpointPreset]: ... + @abstractmethod + async def get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: ... + + @abstractmethod + async def delete_preset(self, project_name: str, model: str) -> None: ... + + @abstractmethod + async def save_preset(self, project_name: str, preset: EndpointPreset, comments=None) -> EndpointPreset: ... + class LocalDirEndpointPresetService(EndpointPresetService): """Reads *.yml/*.yaml from settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets". Re-reads per call; file IO via run_async. Invalid files are logged and skipped, never fatal.""" -class EndpointPlanReplicaSpecGroup(CoreModel): - name: str - resources: ResourcesSpec - tested_resources: list[ResourcesSpec] - class EndpointPlanJobOffers(CoreModel): replica_group: str offers: list[InstanceOfferWithAvailability] # capped by max_offers @@ -586,6 +669,7 @@ class EndpointPlanJobOffers(CoreModel): @dataclass(frozen=True) class EndpointPresetPlan: # planning.py, not presets.py preset: EndpointPreset + recipe: EndpointPresetRecipe run_plan: RunPlan ``` @@ -597,24 +681,24 @@ This is endpoint planning/orchestration logic, not `EndpointPresetService` stora `planning.py::find_matching_preset_plan(session, project, user, endpoint_name, endpoint_conf, preset_service=None) -> Optional[EndpointPresetPlan]`: 1. Candidates: presets whose top-level `model` equals `endpoint_conf.model` case-insensitively, in sorted file-name order. If there are no candidates, return `None` without refreshing the user's SSH key or calling the run planner. -2. For each candidate, build the merged service config (below), wrap in a `RunSpec` (repo fields `None` → virtual repo; `RunSpec.profile` is Optional), and call `runs.get_plan(session, project, user, run_spec, max_offers=1)`. Use the endpoint's effective `creation_policy`; if omitted, keep the normal run default `reuse-or-create` so elastic fleets may provision new instances. If the user explicitly sets `creation_policy: reuse`, matching becomes existing-instances-only. Match ⇔ every `job_plan` has ≥1 offer with `offer.availability.is_available()`. +2. For each candidate recipe, build the merged service config (below), wrap in a `RunSpec` (repo fields `None` → virtual repo; `RunSpec.profile` is Optional), and call `runs.get_plan(session, project, user, run_spec, max_offers=1)`. Use the endpoint's effective `creation_policy`; if omitted, keep the normal run default `reuse-or-create` so elastic fleets may provision new instances. If the user explicitly sets `creation_policy: reuse`, matching becomes existing-instances-only. Match ⇔ every `job_plan` has ≥1 offer with `offer.availability.is_available()`. - Before plan/submission, mirror the run router's behavior: if the creator user has no `ssh_public_key`, call `users.refresh_ssh_key(...)` rather than failing the endpoint. This keeps endpoint apply consistent with `dstack apply` for services. - **Wrap each candidate in `try/except ServerClientError` → skip**: `get_plan` validates the merged config and can raise for a bad preset; one bad preset must not abort matching. - Cost note: the planner enumerates cloud backend offers per candidate cloud fleet (`plan.py::get_job_plans` → `find_optimal_fleet_with_offers`) — each candidate evaluation can take seconds. This is acceptable in `/get_plan` and the pipeline only because matching filters by model before planning and uses `max_offers=1`; keep the local preset set small and do not broaden matching into a registry scan in v1. - - Known under-check, accepted for v1 (matching is advisory): `runs.get_plan` produces one representative `JobPlan` per replica group (`replica_num=0`), while the preset may record multiple tested resources per group. V1 verifies each group has at least one available matching offer for its `resources` and leaves full cardinality/capacity checks to the run scheduler. Capacity-aware matching over every tested replica is Later. + - Known under-check, accepted for v1 (matching is advisory): `runs.get_plan` produces one representative `JobPlan` per replica group (`replica_num=0`), while validations may record multiple exact running replicas per group. V1 verifies each group has at least one available matching offer for the recipe service resources and leaves full cardinality/capacity checks to the run scheduler. Capacity-aware matching over every validation replica is Later. 3. First match wins. -`EndpointPresetPlan` contains the selected `EndpointPreset` and the `RunPlan` computed from the merged `RunSpec`. The caller must submit that same plan/spec path rather than rebuilding independently, mirroring the run apply split between `runs.get_plan` and `runs.apply_plan`. +`EndpointPresetPlan` contains the selected `EndpointPreset`, selected `EndpointPresetRecipe`, and the `RunPlan` computed from the merged `RunSpec`. The caller must submit that same plan/spec path rather than rebuilding independently, mirroring the run apply split between `runs.get_plan` and `runs.apply_plan`. Merged service config (preset → endpoint overrides): -- start from the preset's compiled `ServiceConfiguration` (`service` plus resources derived from the ordered `replica_spec_groups` at the proper service/replica-group level); +- start from the selected recipe's `ServiceConfiguration`; - `name = get_endpoint_serving_run_name(endpoint.name)` (the backing service run name); - merge `endpoint.env` **over** preset env (`Env.update`); endpoint env arrives fully resolved (sentinels rejected at create); - copy every non-`None` `ProfileParams` field from `EndpointConfiguration` onto the service config (both inherit `ProfileParams`, so this is a field-loop like `RunSpec._merged_profile`, `core/models/runs.py:590-607`); -- do **not** merge or override resources from the endpoint: endpoints do not expose resources in v1, and the preset's replica group `resources` are the tested scheduling requirements; +- do **not** merge or override resources from the endpoint: endpoints do not expose resources in v1, and the recipe service resources are the tested scheduling requirements; - do not force `creation_policy = reuse`: the normal default `reuse-or-create` is intentional because existing dstack fleets may be elastic and allowed to provision new instances for submitted services/jobs. -How this runs without the agent: the local preset loader injects `replica_spec_groups[0].resources` into top-level `service.resources` for the implicit group, or injects `replica_spec_groups[i].resources` into `service.replicas[i].resources` for explicit replica groups. It also leaves `service.replicas`/`scaling` intact, so the normal service planner/submission path decides how many replicas to start and where to place them. `len(replica_spec_groups[i].tested_resources)` records the verified running replica count at save time; it is evidence and future inspect/JSON input, not a replacement for service `replicas`/autoscaling. +How this runs without the agent: the selected recipe service is already a normal `ServiceConfiguration` without a name/profile. The endpoint layer sets the backing service name and endpoint/profile constraints, then normal run planning/submission decides how many replicas to start and where to place them. `validations` record verified running replica counts/resources at save time; they are evidence and future inspect/JSON input, not a replacement for service `replicas`/autoscaling. Submission: `RunSpec(run_name=get_endpoint_serving_run_name(endpoint.name), configuration=merged, ssh_key_pub=None)` as the creator user, using the same policy/SSH-key behavior as the run router (`refresh_ssh_key` if needed; do not let `get_plan` and the final run submission see different specs). If validation still fails, surface that as endpoint FAILED. @@ -625,9 +709,9 @@ Fleet-drift note: matching is advisory — a fleet can become busy between match When an agent-created backing service becomes `RUNNING` through the normal service readiness path, save a sanitized `endpoint-preset` wrapper back into the endpoint project's local preset directory (`/projects//presets`). This makes the second endpoint for the same model in the same project take the deterministic preset path instead of paying the agent again. Implementation details: -- Extend `EndpointPresetService` with project-scoped operations such as `save_preset(project_name, preset, comments) -> EndpointPreset`. The local implementation writes a `type: endpoint-preset` YAML file under `settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets"` using an atomic temp-file rename. File names should be stable and collision-resistant, e.g. `-.dstack.yml`; do not overwrite hand-authored presets. -- Source of truth is the backing run's stored `RunSpec.configuration` plus the latest successful job submissions after the service has reached `RUNNING`, not the agent's final text. Sanitize before writing: top-level `model` comes from the verified service model; `replica_spec_groups[*].resources` is copied from the verified service configuration's per-group scheduling requirements; `replica_spec_groups[*].tested_resources` are built from jobs grouped by `JobSpec.replica_group` in `ServiceConfiguration.replica_groups` order, deriving each exact instance `ResourcesSpec` from `JobRuntimeData.offer` when present or the backing `InstanceModel.offer` as fallback. The `service` section preserves the serving recipe/shape fields (`image`, `commands`, `port`, `model`, `volumes`, `replicas`, probes, scaling) but clears `type`, `name`, all resource fields, and `ProfileParams`. Merge env as names only for endpoint-provided keys and redact secret-looking values (`TOKEN`, `KEY`, `SECRET`, `PASSWORD`) to name-only entries. Never write resolved secret values from `EndpointConfiguration.env` to disk. -- Preserve provenance as YAML comments at the top (comments do not affect preset parsing): endpoint name/id, run id, dstack version, timestamp, agent model, job ids that produced the replica spec groups, and recipe source URLs from the agent's structured final report. Do not add a metadata object in v1. +- Extend `EndpointPresetService` with project-scoped operations such as `save_preset(project_name, preset, comments) -> EndpointPreset`. The local implementation writes a `type: endpoint-preset` YAML file under `settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets"` using an atomic temp-file rename. File names are derived from the model; lookup and merge are by `model`. +- Source of truth is the backing run's stored `RunSpec.configuration` plus the latest successful job submissions after the service has reached `RUNNING`, not the agent's final text. Sanitize before writing: top-level `model` comes from the verified service model; `recipes[*].service` preserves the serving recipe/shape fields (`image`, `commands`, `port`, `model`, `resources`, `volumes`, `replicas`, probes, scaling) but clears `type`, `name`, and `ProfileParams`; `recipes[*].validations[*].replicas[*].resources` are built from jobs grouped by `JobSpec.replica_group` in `ServiceConfiguration.replica_groups` order, deriving each exact instance `ResourcesSpec` from `JobRuntimeData.offer` when present or the backing `InstanceModel.offer` as fallback. Merge env as names only for endpoint-provided keys and redact secret-looking values (`TOKEN`, `KEY`, `SECRET`, `PASSWORD`) to name-only entries. Never write resolved secret values from `EndpointConfiguration.env` to disk. +- Keep YAML metadata minimal in v1. Comments may note the generating endpoint/run, but there is no structured metadata object until a concrete reader uses it. - Failure to save the preset logs a warning and emits an event, but it does **not** fail an already-running endpoint. Tests should cover successful write, duplicate-name avoidance, invalid destination permissions, and secret redaction. --- @@ -669,9 +753,9 @@ class AgentService(ABC): What this means for v1: - *Runtime hardening through live runs*: keep the Claude Code subprocess path intentionally simple, then harden it based on real endpoint attempts. It must operate non-interactively, use the real `dstack` CLI, create/edit files, inspect logs/status, preserve artifacts, emit a final machine-readable report, avoid duplicate live agent processes after restarts, and support endpoint stop aborts. - *CLI-first dstack operations*: planning, submission, status inspection, log reading, and stopping are done through the same `dstack` CLI a user would run: `dstack apply`, `dstack ps`, `dstack run get --json`, `dstack logs`, and `dstack stop`. Do not build duplicate custom tools named `submit_service`, `get_run_status`, `get_run_logs`, etc. -- *Termination contract*: the agent returns a structured final report with success/failure, final candidate run name/id, final service YAML/content, recipe sources, and verification summary. The agent is responsible for final functional verification. The server does not trust the report blindly; it validates the claimed run identity and project/user ownership before linking the run, then waits for normal dstack service readiness before activating the endpoint. The readiness gate is bookkeeping, not a server-side claim that the model works. +- *Termination contract*: the agent returns a structured final report with success/failure, final service run name/id, final service YAML/content, recipe sources, and verification summary. The agent is responsible for final functional verification. The server does not trust the report blindly; it validates the claimed run identity and project/user ownership before linking the run, then waits for normal dstack service readiness before activating the endpoint. The readiness gate is bookkeeping, not a server-side claim that the model works. - *Environment hygiene*: the agent runtime receives only the credentials it needs to authenticate to Claude plus the isolated dstack CLI config for the target project. The command/workspace environment remains scrubbed: do not expose server DB credentials, encryption keys, cloud backend credentials, or unrelated server env vars. -- *Abortability*: endpoint stop must be able to stop or ask the agent runtime to stop. This is not complete in the v0 subprocess runtime yet: the worker can hold the endpoint lease while Claude runs, and stop-time cancellation/restart-safe recovery must be hardened before production use. +- *Abortability and restart*: endpoint stop aborts same-host Claude process groups and stops linked/submitted runs through endpoint submission rows. Same-host restart/resume reuses the existing agent session/workspace. Multi-host restart/supervision, duplicate-process prevention across server instances, and final handoff edge cases still need hardening before production use. Packaging requirement: agent runtime dependencies must be installed automatically by normal server deployment paths. The server runtime must not depend on `uv` being available. The Claude Agent SDK currently depends on pydantic v2 while dstack still runs pydantic v1, so v0 invokes the Claude Code executable directly instead of importing the SDK into the server process. Docker release and staging images copy the bundled Claude Code binary into `/usr/local/bin/claude` at build time. `DSTACK_AGENT_CLAUDE_PATH` may point the server at a specific Claude Code executable; when it is unset, `ClaudeAgentService` resolves `claude` from `PATH`. For non-Docker installs, automatic non-Docker packaging for that executable remains a release-blocking follow-up before the agent path can be considered production-ready. @@ -688,30 +772,39 @@ The important product work is the harness, not the HTTP loop. The agent should b 2. gather grounded recipe + hardware evidence from credible sources; 3. inspect the project context and available fleet/offer envelope through CLI/status output; 4. draft the smallest service YAML that should work; -5. preview with `dstack apply -f ` before spending GPU time; -6. submit detached with `dstack apply -f -y -d` only when the plan is acceptable within the endpoint/profile/budget envelope; -7. debug with `dstack ps`, `dstack run get --json`, and `dstack logs`; -8. stop bad candidates with `dstack stop -y` and let normal run lifecycle handle termination/deletion; -9. finish only after the final service is RUNNING, has a registered replica, exposes a model URL, and the agent has made a final model request proving the requested model is actually served. +5. choose the experiment type deliberately: dev environment, task, or service; +6. preview with `dstack apply -f ` before spending GPU time; +7. submit detached with `dstack apply -f -y -d` only when the plan is acceptable within the endpoint/profile envelope; +8. debug with `dstack ps`, `dstack run get --json`, and `dstack logs`; +9. stop ruled-out submitted runs with `dstack stop -y` and let normal run lifecycle handle termination/deletion; +10. finish only after the final service is RUNNING, has a registered replica, exposes a model URL, and the agent has made a final model request proving the requested model is actually served. This is intentionally **not** a new server-side mini API for dstack. The harness should provide: - an endpoint-scoped working directory for service YAMLs, notes, and command transcripts; - the real `dstack` binary from the running installation; - a CLI config/context that targets the correct server, project, and endpoint creator user; -- endpoint constraints: model, env names, ProfileParams, preset policy, and allowed resource/spend envelope; +- endpoint constraints: model, env names, ProfileParams, preset policy, and allowed fleet/profile envelope; - recipe grounding guidance and optionally preloaded recipe/context snippets, not a required `find_model_recipes` tool; - command logging and a `should_abort()` check between command iterations so deletion can interrupt long agent work; - a structured final-report schema. -The preset path uses a server-owned deterministic run name from `get_endpoint_serving_run_name(endpoint.name)` because the server submits that service itself. The agent path does not force that naming convention. Claude may choose concise, unique candidate and final run names; the server validates and links the final service by reported run ID, then records it in `EndpointModel.service_run_id` and `EndpointRunSubmissionModel`. +The preset path uses a server-owned deterministic run name from `get_endpoint_serving_run_name(endpoint.name)` because the server submits that service itself. The agent path uses a strict submission naming contract for every dstack run Claude creates: `-1`, `-2`, and so on. Claude still decides what each run is for; framework, hardware, purpose, and role details belong in `submissions.jsonl`, progress messages, `verification.json`, and the final report, not in the run name. The server validates and links the final service by reported run ID, then records it in `EndpointModel.service_run_id` and `EndpointRunSubmissionModel`. + +Dev-environment and task prototyping are not a side path; they are part of the intended harness. The agent should submit a service directly only when the recipe/image/command/resources are already known enough that URL wiring and final service behavior are the main remaining question. If failures show package/runtime compatibility, launch flags, model load, driver/CUDA mismatch, or backend provisioning uncertainty, the next experiment should often use a reusable fleet path, dev environment, or small task when that can avoid repeated cold provisioning. Warm idle instances can speed services too, not only dev environments. This remains agent judgment, not a hardcoded retry rule. For container-only backends that cannot reuse instances, the agent may still use tasks/services, but should avoid resubmitting the same service shape just to learn basic runtime facts. A probe only counts if it exercises the intended serving stack deeply enough to justify promotion; a host-only check such as `nvidia-smi` is not enough. The clean service is still the final proof, and failed final service verification must send the agent back into the evidence loop instead of producing success. + +Endpoint agents do **not** create fleets. This is a product contract, not just prompt preference: the endpoint apply plan and server create path require at least one active existing fleet usable by the endpoint. By default the agent may consider all existing project/imported fleets; endpoint `fleets` narrows that set. If no active fleet exists, the plan/create path reports "The project has no fleets. Create one before submitting an endpoint." If `fleets` is set but none match, it reports that no fleets match the endpoint configuration. + +Within that contract, fleet/backend/hardware choice is still the agent's decision. It should inspect existing fleets, pass applicable `--fleet` filters to `dstack offer`, plan previews, and submitted runs, and choose among the allowed fleets based on the model, endpoint/profile constraints, offer envelope, current fleet state, and experiment. The agent must not create, delete, apply, or edit fleets, including `nodes`, `target`, `idle_duration`, backends, resources, max nodes, or ownership. If the allowed fleets cannot support a useful experiment, it should fail with clear evidence for the user/admin. + +Driver/runtime compatibility is a first-class uncertainty. Some backends expose different host NVIDIA driver/CUDA compatibility across provider pools, regions, GPU generations, or individual nodes, and dstack offers/fleets may not expose that version. The agent may estimate likelihood from provider/GPU/region evidence, but when CUDA/runtime compatibility matters it should gather evidence (`nvidia-smi`, run logs, dev environment/task probe, or known-compatible image/package sources) before trusting unpinned framework installs. This evidence belongs in agent artifacts and future validation metadata; it must not become a fake scheduling constraint unless the backend/user constraints can actually enforce it. -Dev-environment and task prototyping are allowed in the v0 agent prompt when they are the fastest way to reduce uncertainty about an image, launch command, model download, or hardware choice. For v1, advanced P/D disaggregation, multi-service routers/workers, load benchmarking, and autoscaling tuning are Later unless needed to make the requested model serve at all. +For v1, advanced P/D disaggregation, multi-service routers/workers, load benchmarking, and autoscaling tuning are Later unless needed to make the requested model serve at all. ### 8.4 Prompt, recipe grounding & vendored context Runtime context layout: -- `resources/system_prompt.md` — endpoint-specific mission and protocol only: use the real CLI, load `/dstack` and `/dstack-prototyping`, honor endpoint constraints, emit concise progress events, verify the final model API request, and return the structured final report. +- `resources/system_prompt.md` — endpoint-specific mission and protocol only: use the real CLI, load `/dstack` and `/dstack-prototyping`, honor endpoint constraints, emit free-form natural-language progress updates, verify the final model API request, and return the structured final report. - repo-root `skills/dstack/SKILL.md` — CLI/config source of truth. - repo-root `skills/dstack-prototyping/SKILL.md` — reusable research-to-working-workload skill: source order, model-serving recipe selection, hardware fit, dev-environment/task/service experiment choice, failure classification, final service cleanup, and model API verification. @@ -721,16 +814,16 @@ Runtime context layout: Recipe grounding should be source-oriented, not a static recipe encyclopedia. The agent should prefer current primary sources such as model cards, vLLM/SGLang docs, dstack docs/CLI help, and its own command/log evidence. Advanced posts such as Wafer GLM-on-AMD and LMSYS agent-assisted SGLang development are directional for future harness work, not v1 requirements. -The prompt interpolates endpoint env keys (names only), ProfileParams constraints, project context, and effective agent budget. Recipe grounding is discovered by the agent through allowed network/command facilities and recorded in the final report; do not model it as a required `find_model_recipes` function in v1. +The prompt interpolates endpoint env keys (names only), ProfileParams constraints, and project context. Recipe grounding is discovered by the agent through allowed network/command facilities and recorded in the final report; do not model it as a required `find_model_recipes` function in v1. ### 8.5 Execution, cost & limits - Runs inside the pipeline worker (§6.2) unless the chosen runtime forces a detached process model. Agent execution must not block the event loop; use async subprocess/process supervision or a small dedicated executor with bounded concurrency, not the shared 128-thread default executor. - Between agent/runtime steps, or through runtime cancellation hooks, the service checks `should_abort()` (cheap SELECT of stop intent + `lock_token` sanity) and exits early on stop. -- Do **not** add generic endpoint provisioning timeouts or unused turn/attempt counters. The real agent loop should introduce budgets only where they are enforced. v0 uses the endpoint configuration's `max_agent_budget` value, falling back to `DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD`, and passes the effective value to Claude Code's `--max-budget-usd` as the hard per-agent-process cost cap. Once retries/resumes or multiple Claude processes per endpoint provisioning are added, dstack must persist and sum agent spend per endpoint provisioning attempt before starting the next process, so the total endpoint provisioning budget cannot be bypassed by restarts. +- Do **not** add generic endpoint provisioning timeouts, unused turn counters, or a public agent-budget field in v1. Agent budget/cost governance is Later and must be added only with durable per-session accounting and an actually enforced runtime contract. - Model via `DSTACK_AGENT_ANTHROPIC_MODEL` if the selected runtime supports explicit model selection (default: `claude-opus-4-8`; the default agent should bias toward the strongest Anthropic model because endpoint provisioning is an expensive, tool-heavy deployment investigation). - Concurrency bound = pipeline `workers_num` per replica (4). -- Observability: log each command + final summary at INFO with the endpoint id; store command transcripts under the endpoint workspace or log service for debugging; store the final agent summary in `status_message` on failure; log provider usage for token-budget accounting; store `recipe_sources` in the saved preset comments on success. Cost accounting/events → Later. +- Observability: log each command + final summary at INFO with the endpoint id; store command transcripts under the endpoint workspace or log service for debugging; store the final agent summary in `status_message` on failure; store `recipe_sources` in the saved preset comments on success. Cost accounting/events → Later. --- @@ -743,7 +836,6 @@ New in `src/dstack/_internal/server/settings.py` (documented in `mkdocs/docs/ref | `DSTACK_AGENT_ANTHROPIC_API_KEY` | `AGENT_ANTHROPIC_API_KEY` | `None` | as specified by the requirement (agent-scoped, hence no `_SERVER_`); presence ⇒ agent enabled | | `DSTACK_AGENT_CLAUDE_PATH` | `AGENT_CLAUDE_PATH` | `None` | optional path to the Claude Code executable; falls back to resolving `claude` from `PATH` | | `DSTACK_AGENT_ANTHROPIC_MODEL` | `AGENT_ANTHROPIC_MODEL` | `claude-opus-4-8` | override only if the operator intentionally wants a cheaper/faster model | -| `DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD` | `AGENT_ANTHROPIC_MAX_BUDGET` | `None` | optional server default for endpoint agent spend; overridden by endpoint `max_agent_budget`; future retry/resume support must sum spend across processes per endpoint provisioning attempt | Plain module constants in `settings.py`, not env-configurable in v1: `SERVER_PROJECTS_DIR_PATH = /projects`, `ENDPOINT_RUNNING_CHECK_INTERVAL = 60s`. @@ -757,7 +849,7 @@ Docs: `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._inte **M1 — resource skeleton (no processing).** `core/models/endpoints.py` + registration in `configurations.py`; `EndpointModel` + migration + events wiring; `EndpointPlan` models with the `none` provisioning branch; `services/endpoints/__init__.py` (create/stop/list/get/get_plan shell, status switch, events); schemas + router + `app.py`; `_endpoints.py` API client group; CLI configurator + `EndpointCommand`. Endpoints can be planned/created and sit SUBMITTED forever. Tests: router CRUD, name uniqueness/reapply semantics, configurator parse, sentinel rejection, plan output for no preset/no agent. **M2 — pipeline & lifecycle.** `background/pipeline_tasks/endpoints.py` + registration; the full state machine of §6.2 with `AgentService`/`EndpointPresetService` as interfaces with disabled/empty defaults (so: SUBMITTED→FAILED "nothing configured", preset-path run-name conflict detection, RUNNING backing-run liveness through `service_run_id`, simple two-phase server-side stop of the linked service run, and FAILED teardown rule). Tests: fetcher query (sqlite+postgres, lock claims, RUNNING cadence, STOPPING handling), every worker transition with fakes, conflict detection, linked-run cleanup. Do not test name-based adoption as ownership. -**M3 — presets + plan offers.** `presets.py` (`EndpointPresetService`, `LocalDirEndpointPresetService`, project-scoped storage/parsing only under `/projects//presets`), `endpoint-preset` wrapper parsing (`service` recipe + ordered tested `replica_spec_groups`), `planning.py` matching via `get_plan` using the endpoint's effective `creation_policy` (default `reuse-or-create`, so elastic fleets are allowed), per-candidate `ServerClientError`/unresolved-env skip, config merge, submission path, `EndpointPlan.provisioning_plan=type:"preset"` only for provisionable presets with tested replica spec groups and offer summary, save-preset interface + sanitizer. Tests: matching unit tests with testing factories (project/fleet/instance), project isolation, merge semantics, invalid preset files skipped, bad-preset-skip, no-offer preset falls through to agent when policy allows, plan prints selected preset/offers/replica spec groups, secret redaction, atomic write/no-overwrite. +**M3 — presets + plan offers.** `presets.py` (`EndpointPresetService`, `LocalDirEndpointPresetService`, project-scoped storage/parsing only under `/projects//presets`), `endpoint-preset` wrapper parsing (`recipes[*].service` + ordered `validations`), legacy preset conversion, `planning.py` matching via `get_plan` using the endpoint's effective `creation_policy` (default `reuse-or-create`, so elastic fleets are allowed), per-recipe `ServerClientError`/unresolved-env skip, config merge, submission path, `EndpointPlan.provisioning_plan=type:"preset"` only for provisionable recipes with offer summary, save-preset interface + sanitizer/merge. Tests: matching unit tests with testing factories (project/fleet/instance), project isolation, merge semantics, invalid preset files skipped, bad-preset-skip, no-offer preset falls through to agent when policy allows, plan prints selected preset/recipe/offers, secret redaction, atomic write. **M4 — agent + endpoint logs.** Build on `service_run_id` + `EndpointRunSubmissionModel` so only the latest run serves while submission history is preserved. Continue hardening `ClaudeAgentService` around the v0 Claude Code subprocess runtime: workspace/process handling, scrubbed environment, structured final report artifact, vendored context, `EndpointPlan.provisioning_plan=type:"agent"`, agent settings, automatically installed runtime dependencies, `should_abort`/process cancellation, successful-agent preset save on `RUNNING`, `dstack endpoint logs`, and non-detached endpoint apply following server-side logs/status without attach. Tests should cover real runtime integration boundaries actually used by `ClaudeAgentService`, final-report parsing, runtime availability packaging checks for the `server`/`all` extras, endpoint logs command behavior, apply does not call attach, fake-runtime service integration, and FakeAgentService pipeline integration; mocked network/CLI where possible. Manual e2e: local `uv` server install plus server Docker image both reach the same runtime availability state with only `DSTACK_AGENT_ANTHROPIC_API_KEY` configured. **M5 — docs & polish.** Reference page, env.md, concepts page, `dstack endpoint` help texts, endpoint plan rendering polish, example presets in docs (not shipped as defaults — §11 Q7), manual e2e runbook (local server + real fleet + one preset + one agent deploy). @@ -765,9 +857,9 @@ Docs: `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._inte ## 11. Key open questions — with recommended answers -1. **Run naming & history.** Q: how to link endpoint↔run and what happens on retries? **A:** preset-backed services use `-serving` when valid because the server submits them. Agent-backed services are linked by reported `run_id`; the agent chooses concise unique run names. `service_run_id` is authoritative for the serving run; name lookup is conflict detection only for preset submission. `EndpointRunSubmissionModel(endpoint_id, run_id, submission_num, submitted_at)` records endpoint-submitted run history without coupling `RunModel` to endpoints. Agent retry policy → Later. +1. **Run naming & history.** Q: how to link endpoint↔run and what happens on retries? **A:** preset-backed services use `-serving` when valid because the server submits them. Agent-backed dstack runs use numbered endpoint submissions (`-1`, `-2`, ...), while the final serving run is linked by reported `run_id`. `service_run_id` is authoritative for the serving run; name lookup is conflict detection only for preset submission and strict submission reconciliation in the agent path. `EndpointRunSubmissionModel(endpoint_id, run_id, submission_num, submitted_at)` records endpoint-submitted run history without coupling `RunModel` to endpoints. Agent retry policy → Later. 2. **Who owns the backing run?** Q: which UserModel submits it (no system user exists)? **A:** the endpoint's creator (`user_id` on the row) — correct attribution in events/quotas; mirror the run router by refreshing the user's server-managed SSH key before `get_plan`/submission if missing. A first-class service account → Later. -3. **Where does the agent execute?** Q: inside the pipeline worker vs a detached task with its own heartbeat bookkeeping? **A:** v0 runs inside the worker `process()` under the Heartbeater lease, with `workers_num=4` as the concurrency bound. This is enough to test the real loop, but stop-time cancellation and restart-safe recovery are not done: `stop_endpoints` can mark the endpoint `STOPPING` without acquiring the lock, but the live Claude subprocess still needs an explicit abort/supervision path. +3. **Where does the agent execute?** Q: inside the pipeline worker vs a detached task with its own heartbeat bookkeeping? **A:** v0 runs inside the worker `process()` under the Heartbeater lease, with `workers_num=4` as the concurrency bound. Same-host process abort and same-session restart/resume are implemented, which is enough to keep testing the real loop. Multi-host supervision and final handoff recovery remain hardening items. 4. **Which Claude runtime?** **A: v0 uses Claude Code headless/subprocess after rejecting the raw `anthropic` Messages loop.** Continue validating non-interactive server execution, cancellation, workspace/env isolation, final artifact extraction, restart behavior, and packaging. If Claude Code proves unsuitable after real deployments, choose another real agent runtime before adding more harness complexity. 5. **Permissions.** Q: `ProjectMember` (volumes) or `ProjectAdmin` (gateways)? **A:** `ProjectMember` — endpoints are project workloads that consume the same quota surface as runs, not shared admin infrastructure. 6. **Preset match semantics.** Q: what does "matches the existing fleets" mean concretely? **A:** A model-matched preset is only **provisionable** when `get_plan` on the merged config returns ≥1 available offer for every job plan using the endpoint's effective `creation_policy`. Default `reuse-or-create` may return offers from elastic fleets that can create new instances; explicit `reuse` restricts to currently existing instances. Only provisionable presets are submitted automatically. If presets match the model but no offers are available, `preset_policy: reuse` stops with no-offers/no-fleets, while `reuse-or-create` falls through to the agent path if available. Advisory, not a binding; known to under-check multi-replica capacity (§7.3), accepted for v1. @@ -783,15 +875,17 @@ Docs: `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._inte Carried over from the requirements ("later we…") plus deferrals made above: -- **Preset sources beyond local dir**: S3, git repo (reuse the `services/templates.py` clone/TTL machinery), HTTP registry; richer provenance metadata (version pins, observed offer, benchmark results), signed/curated preset channels; shipping default presets; capacity-aware matching over every recorded replica spec and exact offer identity. +- **Preset sources beyond local dir**: S3, git repo (reuse the `services/templates.py` clone/TTL machinery), HTTP registry; richer validation metadata (version pins, observed offer, benchmark results), signed/curated preset channels; shipping default presets; capacity-aware matching over every recorded validation replica and exact offer identity. - **Preset update policy**: allow the agent to update/replace an existing learned preset only as a repair path, not as optimization. The harness must first prove that the old preset is not reproducible under the conditions it claimed to support (for example same model family/recipe constraints and compatible tested resource topology, but repeated normal preset-path attempts fail for reasons attributable to the preset rather than transient capacity/no-offers/user constraints). Only then may it save a replacement or new version, with evidence of the failed reproducibility attempt and the newly verified deployment. V1 learned presets stay append-only/no-overwrite until this policy is designed and tested on real failures. -- **Automatic provisioning retry policy**: retry failed provisioning through SUBMITTED, backoff, retry history retention, and distinct candidate run names if preserving failed candidates becomes important. +- **Explicit recipe selection**: allow endpoint configs to specify `recipe: ` later when a model has multiple learned recipes. V1 does not expose this field; it tries stored recipes in order and uses the first recipe with available offers under the endpoint's effective fleets/profile constraints. +- **Automatic provisioning retry policy**: retry failed provisioning through SUBMITTED, backoff, retry history retention, and distinct submitted run names if preserving failed submissions becomes important. - **Agent retry policy for RUNNING endpoints**: instead of FAILED on health-check failure, re-invoke the agent to diagnose/redeploy; backoff policy; `retry` ProfileParams semantics for endpoints. - **Alternative agent runtime** — revisit if the first real runtime cannot meet server requirements for non-interactive execution, packaging, cancellation, workspace/env isolation, artifact capture, and cost accounting. - **Other `AgentService` implementations** (OpenAI-compatible, Bedrock/Vertex via the SDK's provider clients, self-hosted). - **Richer service creation**: deepen `dstack-prototyping` through real endpoint-agent traces; richer attach/SSH automation for dev environments; benchmarking before marking an endpoint running; autoscaling/replicas/gateway/domain decisions; quantization variant selection; multi-node deployments; P/D disaggregation and other multi-component serving topologies; advanced SGLang development/deployment harnesses inspired by the 2026-07-02 LMSYS agent-assisted SGLang workflow. -- **Workload-aware optimization**: benchmark and tune endpoints by workload profile (chatbot, RAG, code generation, long-form generation), measuring TTFT, ITL, throughput, end-to-end latency, quality, and cost/token. Product references such as Modal Auto Endpoints, Makora, and Runpod Overdrive are useful directionally, but v1 must stop at verified functional deployment and reproducible preset provenance. +- **Workload-aware optimization**: benchmark and tune endpoints by workload profile (chatbot, RAG, code generation, long-form generation), measuring TTFT, ITL, throughput, end-to-end latency, quality, and cost/token. Product references such as Modal Auto Endpoints, Makora, and Runpod Overdrive are useful directionally, but v1 must stop at verified functional deployment and reproducible preset recipes. - **In-place update / plan-apply**: server-side endpoint `get_plan`/`apply_plan`, model or config changes via the existing service-run `apply_plan` + rolling deployment machinery, no stop/recreate for simple changes; endpoint row updates protected by a `configuration_version`/deployment guard for multi-server safety. +- **Endpoint-level resources**: optional user-specified resource requirements for endpoint configs. Deferred because replica groups make merge semantics non-trivial. If added, the field must be enforced by preset planning and agent-submitted YAML, must define how it combines with preset/service replica-group resources, and must be reflected in the apply plan without pretending it is final tested hardware. - **Deletion escalation**: forced abort / operator-facing stuck-deletion handling if a backing run remains terminating beyond the normal RunPipeline retry window. - **Plugins**: add an `EndpointSpec` to the `ApplySpec` TypeVar (`dstack/plugins/_models.py:8`) so plugin policies can reason about endpoint-level intent directly. Backing service runs still use normal run policies in v1. - **Frontend UI**: endpoints page in the server frontend. No frontend work is required in v1. @@ -810,7 +904,7 @@ Carried over from the requirements ("later we…") plus deferrals made above: | Replica death mid-agent-run duplicates deployments / spend | `service_run_id` points to the latest run and `EndpointRunSubmissionModel` records accepted endpoint-submitted runs. The heartbeater lease prevents duplicate live workers while the replica lives; after a crash, the next worker reconciles by linked run/submission history. Name lookup is conflict-only (§6.2, 6.3, 6.5) | | Paid crash loop (pipeline re-fetch + flaky agent) | no automatic retry in v1 + terminal FAILED stops re-fetching; real agent loop must add explicit token/cost/wall-clock budgets before it can spend on experiments | | Adopting/destroying an unrelated user run with the same name | name lookup is conflict detection only; ownership must come from `service_run_id` or `EndpointRunSubmissionModel` rows, never from name/user/timestamp heuristics (§6.5) | -| One-shot stop+delete of runs always raising | endpoint stop remains two-phase stop → wait-until-finished → stopped. The CLI-agent should use normal `dstack stop -y` for bad candidates and let run lifecycle handle termination (§2.5, §8.3) | +| One-shot stop+delete of runs always raising | endpoint stop remains two-phase stop → wait-until-finished → stopped. The CLI-agent should use normal `dstack stop -y` for ruled-out submitted runs and let run lifecycle handle termination (§2.5, §8.3) | | GPU-run leaks on terminal FAILED | every FAILED transition issues `stop_runs` for a still non-terminal backing run (§6.2) | | Transaction/lock corruption submitting backing runs from the worker | fresh `get_session_ctx()` for all runs-service calls; endpoint row writes stay token-guarded; any interim progress update leaves lock columns untouched (§6.1) | | Prompt injection via fetched recipes → server compromise | command execution runs in an endpoint-scoped workspace with a scrubbed environment; the agent uses normal `dstack` CLI rather than server internals; server secrets/DB/cloud credentials and the agent API key are not exposed to commands (§8.2, 8.3) | @@ -820,6 +914,8 @@ Carried over from the requirements ("later we…") plus deferrals made above: | `MissingGreenlet` on lazy `project.backends` in the worker | refetch joinedloads chain `project → backends` (§6.1) | | `register_service` raises synchronously during backing run submission (gateway config issues) | caught as submission failure ⇒ FAILED with message; agent prompt includes gateway context | | Agent deploys a service without `model:` → no model URL to surface | prompt requires `model:`; agent final verification must use the model endpoint, and server readiness validation refuses to mark the endpoint running unless the backing service exposes `ServiceSpec.model.base_url` (§6.2, §8.3) | +| Task-first degenerates into a batch script | prompt/skill require long-lived interactive probes when attach/SSH is available, and the endpoint agent's local `dstack` wrapper now rejects batch-style probe tasks before submission. Unit coverage verifies rejection of a task that packs host/framework/server/curl checks into `commands` and acceptance of a `sleep infinity` probe task. Live e2e must still prove the agent reacts correctly by attaching/SSHing into the task and promoting a clean service | +| Repeated model downloads because cache mounts are omitted | prompt/skill now require optional instance cache mounts for Hugging Face-style model-serving probes/services when useful, or an explicit artifact explaining why cache mounts do not help for that backend/model. Presets preserve useful `volumes` from the verified service | | Preset/fleet drift between match and provisioning | matching is advisory; run scheduling is source of truth | | Agent runtime dependency missing in deployment | selected runtime dependencies are included in normal server install paths and Docker images; release Docker uses `dstack[all]`, staging uses `uv sync --extra all`, and local server installs must not require manual runtime installation (§8.2, 9) | | Missing migration passes unit tests silently | migration is an explicit M1 review item; postgres-parametrized tests | diff --git a/endpoint-implementation-research/dstack-prototyping-skill-audit.md b/endpoint-implementation-research/dstack-prototyping-skill-audit.md new file mode 100644 index 0000000000..8ae6964cf6 --- /dev/null +++ b/endpoint-implementation-research/dstack-prototyping-skill-audit.md @@ -0,0 +1,115 @@ +# dstack-prototyping Skill Audit + +Date: 2026-07-07 + +Purpose: rebuild `skills/dstack-prototyping/SKILL.md` from validated, high-value +instructions only. This file is a working audit for this branch, not user docs. + +## Skill Bar + +A sentence stays in the skill only if it passes at least one test: + +- It prevents a failure we already observed in endpoint e2e. +- It is a dstack fact verified by docs, CLI help, or source code. +- It is a fragile endpoint-agent contract the model must see during execution. +- It tells the agent how to choose between task, dev environment, service, fleet, + backend, image, resources, cache, or verification. + +A sentence is removed or moved out if it is: + +- generic project-management advice; +- a competitor/product idea that does not change the next deployment decision; +- a future optimization topic unrelated to getting a verified endpoint; +- a claim about backend behavior not supported by docs, code, run evidence, or + harness-provided facts; +- a restatement of `/dstack` CLI syntax unless the endpoint harness depends on + the distinction. + +## Validated Facts + +| Fact | Source | Decision | +| --- | --- | --- | +| dstack has VM-based and container-based backends. | `mkdocs/docs/concepts/backends.md` | Keep. Critical for experiment strategy. | +| VM-based backends give dstack native control over provisioning. | `mkdocs/docs/concepts/backends.md` | Keep, but avoid saying every VM backend is always better. | +| Container-based backends delegate provisioning to the provider or Kubernetes. | `mkdocs/docs/concepts/backends.md` | Keep, but do not overgeneralize to no cache/no SSH without evidence. | +| SSH fleets exist without backend configuration and use user/admin-managed hosts. | `mkdocs/docs/concepts/backends.md`, `mkdocs/docs/concepts/fleets.md` | Keep. Strong signal for interactive prototyping. | +| Fleets must exist before runs. | `mkdocs/docs/concepts/fleets.md`, task/dev/service docs | Keep. Endpoint agent must not create fleets. | +| Backend fleets with `nodes` starting at `0` create only a template; instances are provisioned by runs. | `mkdocs/docs/concepts/fleets.md` | Keep for reasoning about cold starts. | +| Pre-provisioning is supported only for VM-based backends. | `mkdocs/docs/concepts/fleets.md` | Keep. Important distinction from container backends. | +| Fleet `idle_duration` keeps idle backend-fleet instances around for reuse. | `mkdocs/docs/concepts/fleets.md`, snippets | Keep. Explains warm-instance strategy. | +| `dstack offer` ignores fleet configs by default. `--fleet` restricts offers to selected fleets. | `mkdocs/docs/reference/cli/dstack/offer.md` | Keep. Prevents global-offer mistakes. | +| `dstack offer --group-by gpu,backend` is documented and `gpu` must be included. | `mkdocs/docs/reference/cli/dstack/offer.md`, CLI help | Keep. Useful for multi-backend fleets. | +| Offer output is hardware/capacity/price, not proof of backend implementation class. | Inference from offer docs + observed failure | Keep as a guardrail. Label as interpretation, not doc quote. | +| Dev environments are accessible via IDE or SSH. | `mkdocs/docs/concepts/dev-environments.md` | Keep. Useful for interactive probe choice. | +| Tasks run arbitrary commands and can be single-node or distributed. | `mkdocs/docs/concepts/tasks.md` | Keep only the part needed for probes. | +| Services are final endpoint artifacts and can expose OpenAI-compatible model URLs when `model` is set. | `mkdocs/docs/concepts/services.md`, `/dstack` skill | Keep. Final proof depends on it. | +| Resource ranges are supported for CPU/memory/GPU/disk. | task/dev docs, offer reference, CLI help | Keep briefly. | +| GPU spec can include vendor/model/memory/count. | offer reference, dev docs, CLI help | Keep for vendor-aware recipes. | +| Instance volumes can be optional; optional volumes run even if the selected backend cannot mount them. | `mkdocs/docs/concepts/volumes.md` | Keep. Useful for cache mounts without over-constraining placement. | +| Instance volumes are not supported on RunPod and Vast.ai in docs. | `mkdocs/docs/concepts/volumes.md` | Keep carefully; do not extrapolate to all persistence. | +| Local code exposes backend feature lists for create-instance, instance-volumes, privileged, multinode. | `src/dstack/_internal/core/backends/features.py` | Keep if harness injects them. | +| `ComputeWithCreateInstanceSupport` is documented in code as fleet instance creation without running a job; typically VMs implement it and containers do not. | `src/dstack/_internal/core/backends/base/compute.py` | Keep as implementation signal, not public UX. | +| JarvisLabs supports create-instance, privileged, and instance-volumes in this build. | `src/dstack/_internal/core/backends/jarvislabs/compute.py` | Keep in harness hints, not hardcoded in the generic skill. | +| RunPod supports group provisioning, multinode, and network volumes but not create-instance or instance-volumes in this build. | `src/dstack/_internal/core/backends/runpod/compute.py`, features list | Keep in harness hints, not hardcoded in the generic skill. | +| `python` and `image` are mutually exclusive in run configs. | `/dstack` skill | Do not duplicate unless endpoint failures show agents violate it. | + +## Observed Endpoint-Agent Failures To Prevent + +| Failure | Skill/prompt rule | +| --- | --- | +| Agent called JarvisLabs container-style because RunPod rows appeared first in offers. | Classify allowed backends from docs/code/harness facts before relying on offers; offers are not backend-class proof. | +| Agent chose RunPod by omission because service/task YAML did not pin `backends`. | If YAML omits `backends`, admit scheduler may choose any matching backend allowed by constraints. | +| Agent packed a whole probe into task `commands` instead of making an interactive target. | Probe task should stay alive when attach/SSH is available; run checks through attach/SSH. | +| Agent promoted based on shallow host evidence. | `nvidia-smi` alone is not recipe proof. Probe must test the intended image/framework/model/server slice. | +| Agent used `latest` image and hit driver/runtime mismatch. | Final service image should be pinned or justified by source/probe. | +| Agent resubmitted services and paid cold-start cost repeatedly. | Prefer task/dev probes and reusable/inspectable backends when they can reduce total iteration. | +| Endpoint logs had vague categories and little reasoning. | Progress must explain decisions and evidence in natural language. | +| Service reached healthy before endpoint became running. | Final service must still be verified by agent and reported; endpoint success is not service status alone. | + +## Keep In Skill + +- Load `/dstack` first and do not duplicate CLI syntax. +- Define success as verified final service model request. +- Require the agent to decide the proof request, model facts, framework, resource envelope, + constraints, and first experiment before paid runs, without forcing a markdown note. +- Require existing fleets; endpoint agents do not create/edit fleets. +- Use fleet-filtered offers, not global offers, when endpoint capacity is fleet-bounded. +- Compare multi-backend fleets with the intended resource envelope. +- Classify backend choice from docs/code/harness/run evidence, not truncated offers. +- Prefer inspectable/reusable placements when viable, but allow container paths with a reason. +- Derive minimum resources separately from selected offers and validation hardware. +- Use vendor-aware GPU resources after the recipe or placement is vendor-specific. +- Verify driver/runtime when it affects image/framework choice. +- Prefer task/dev probe before final service for new recipes on unverified paths. +- Make probes interactive when attach/SSH is available. +- Probe the intended serving stack, not just GPU visibility. +- Pin final images or justify them from source/probe. +- Use optional instance cache mounts when useful and supported; do not assume persistence. +- Use run JSON/events/logs/attached shell for different evidence types. +- Classify failures before changing YAML. +- Promote only a clean service and verify the model URL with a real request. +- Keep endpoint progress human-readable and structured endpoint files parseable. + +## Remove From Skill + +- Long competitor-link explanations. Keep references in research/test-plan docs; the skill should not push v1 agents toward benchmarking or kernel work. +- Generic "do not chase local optimum" phrasing. Replace with concrete decision gates. +- Full lists of every possible failure unless they guide the immediate next edit. +- Detailed final report schema duplication that belongs in the endpoint prompt. +- Broad claims such as "VM is better" without constraints. +- Any backend-specific fact that can be injected by the harness for the actual project. + +## Rebuild Shape + +The rebuilt `SKILL.md` should be short enough for the agent to actually follow: + +1. Scope and success gate. +2. Pre-run decisions. +3. Fleet/backend choice. +4. Hardware/resources. +5. Experiment selection. +6. Probe quality. +7. Images/cache. +8. Evidence/failure routing. +9. Promotion/final verification. +10. Endpoint progress and structured files. diff --git a/endpoint-implementation-research/dstack-prototyping-skill-outline.md b/endpoint-implementation-research/dstack-prototyping-skill-outline.md new file mode 100644 index 0000000000..ab4a7bfa6d --- /dev/null +++ b/endpoint-implementation-research/dstack-prototyping-skill-outline.md @@ -0,0 +1,11 @@ +# dstack-prototyping Skill Scorecard + +Score every bullet from 0 to 10. Each bullet should stand alone and cover one complete idea. + +- Purpose: use this skill to find a working dstack service recipe for a requested model by testing the recipe in a task on real hardware before submitting the final service. +- Dependency: this skill must be used together with the `dstack` skill; `dstack` provides CLI/YAML syntax, while this skill explains how to prototype model serving with dstack runs. +- Fleet/backend choice: prefer allowed fleets/backends that can leave an idle instance available after a run so repeated task/service attempts can reuse the provisioned machine and local caches, especially model weights; judge this from fleet config, current fleet state, backend docs, and observed run behavior, not from price alone. +- Interactive access: attach, SSH, or Kubernetes exec is useful for inspecting and adjusting a live task, but it is separate from idle instance reuse; prefer both when available. +- Serving sources: check serving-framework sources early enough to choose image, command, flags, resources, cache paths, request format, and expected model behavior; for vLLM and SGLang, treat the listed recipe/docs/cookbook sources as credible starting points. +- Task workflow: before submitting a service, start a long-lived task with `sleep infinity` or equivalent, attach/SSH when available, and test image, installs, model download/cache, serving command, port, launch flags, local model request, and expected model behavior; if image, hardware, or install path changes, submit another task. +- Service workflow: submit the service after the task verifies the recipe, then repeat the model request through the dstack service URL; if the fix requires changing image/install/download/command/resources/cache/model behavior, return to a task, otherwise fix and resubmit the service config. diff --git a/skills/dstack-prototyping/SKILL.md b/skills/dstack-prototyping/SKILL.md index a51e245f88..c043b1022e 100644 --- a/skills/dstack-prototyping/SKILL.md +++ b/skills/dstack-prototyping/SKILL.md @@ -1,241 +1,86 @@ --- name: dstack-prototyping description: | - Use with the dstack skill when a dstack workload, especially model serving, needs research, hardware sizing, recipe selection, live experiments, or debugging before it can be considered ready. Guides how to use dstack dev environments, tasks, services, offers, logs, events, and run JSON as an evidence loop without duplicating dstack CLI/YAML syntax. + Use with the dstack skill for model-serving work when the image, serving command, resources, backend/fleet choice, or service behavior is not proven. Guides task-first prototyping on real hardware, choosing fleets/backends that can reuse idle instances and caches, checking vLLM/SGLang sources, and verifying the final dstack service with a model request. --- # dstack Prototyping -## Required Companion +Use `/dstack` for CLI commands, YAML fields, apply/attach behavior, service URLs, +and other dstack syntax. This skill explains how to use dstack runs while the +model-serving recipe is still unknown. -Load `/dstack` first. `/dstack` is the authority for CLI syntax, YAML fields, confirmation rules, attach behavior, logs, events, offers, fleets, services, and command safety. +## Goal -This skill decides what to investigate, what evidence to collect, when to use each dstack run type, and when a workload is proven. Do not repeat or override `/dstack` command syntax here. If a command or field is uncertain, follow `/dstack`: check help/docs before using it. +Find a working dstack service recipe for the requested model. -## Operating Standard +Before submitting a service, use a task on real hardware to test the serving +image, install/runtime assumptions, model download, cache path, command, port, +launch flags, resources, env vars, backend/fleet choice, and local model +request. Then submit the same recipe as a service and verify the model through +the dstack service URL. -Treat prototyping as an evidence loop: +## Choose Where To Run -1. define the target behavior; -2. build a model/workload/hardware dossier; -3. choose the smallest experiment that can remove the largest uncertainty; -4. run through dstack using `/dstack` rules; -5. classify the result; -6. change only the next unproven assumption; -7. promote only after functional verification. +Choose only VM-based backends, SSH fleets, or Kubernetes fleets because they support idle instances and/or instance volumes. That lets later runs reuse the provisioned/idle instance or instance volumes used by runs for caching model weights (and possibly other writes). You must follow this rule even if there are fleets/backends/offers that are cheaper. The only exception from this rule is when the required GPU class (regardless of the price) is not available through VM-based backend, SSH fleet, or Kubernetes fleet. -Do not chase a local optimum. Step back after each failed candidate and ask whether the problem is the recipe, hardware, image, dstack placement, backend provisioning, auth, model format, or verification method. +Read `https://dstack.ai/docs/concepts/backends.md` to know exactly which +backends are VM-based. -## Source Stack +## Check Serving Sources -Prefer sources that can directly change a deployment decision: +Check serving-framework sources early enough to choose the image, command, +launch flags, resources, cache paths, request format, and expected model +behavior. -1. model card, `config.json`, tokenizer/config files, official repo discussions, and model-family docs; -2. current serving recipe sources: - - `https://recipes.vllm.ai/` - - `https://recipes.vllm.ai/models.json` - - `https://docs.vllm.ai/projects/recipes/en/stable/` - - `https://lmsysorg.mintlify.app/cookbook/intro` - - `https://docs.sglang.ai/` -3. framework docs, release notes, issues, and image tags for vLLM, SGLang, TensorRT-LLM, TGI, or a model-specific server; -4. dstack evidence from offers, plans, run JSON, events, logs, and attached shells; -5. recent deployment writeups when they explain a decision pattern, not just marketing: - - `https://modal.com/blog/introducing-auto-endpoints` - - `https://www.runpod.io/blog/overdrive-benchmarks` - - `https://www.makora.com/` - - `https://www.wafer.ai/blog/glm52-amd` - - `https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development` +For vLLM and SGLang, use these as credible sources: -Use competitor and research links to sharpen the loop: inspectable recipes, workload-specific tuning, engine-level metrics, framework patches, hardware-specific friction, and artifact contracts. Do not treat them as instructions to benchmark, optimize under load, or implement kernels unless the user asked for that scope. +- vLLM recipes and model index: `https://recipes.vllm.ai/` and + `https://recipes.vllm.ai/models.json` +- vLLM recipe docs: `https://docs.vllm.ai/projects/recipes/en/stable/` +- SGLang docs and cookbook: `https://docs.sglang.ai/` and + `https://lmsysorg.mintlify.app/cookbook/intro` -## Dossier Before Spending GPU +Use deeper serving-engine writeups, such as +`https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development`, when +recipes and docs do not explain the model, hardware, or serving failure. -Write a compact note before the first paid run: +## Use A Task Before Service -- requested model and exact served model name; -- target API shape: OpenAI chat, completion, embeddings, multimodal, custom; -- required proof: the smallest request that proves the requested behavior; -- model facts: architecture, parameter count, active parameters for MoE, quantization, context length, tokenizer quirks, license/gating; -- serving candidates and why: vLLM, SGLang, framework recommended by model card, or fallback; -- memory envelope: weight memory, KV cache pressure, tensor/pipeline/data parallel needs, disk cache, CPU/RAM expectations; -- dstack constraints inherited from the request: backends, fleets, spot policy, max price, env names, volumes, regions, or instance types; -- unknowns that require a dev environment, task, or service attempt. +Before submitting a service, start a long-lived task: -If this note cannot name the top two uncertainties, do more source inspection before submitting work. +```yaml +commands: + - sleep infinity +``` -## Hardware Envelope +or an equivalent idle command. -Derive requirements before looking at offers. Offers answer placement, not correctness. +Submit the task detached, attach or SSH into it when available, and run commands +inside the live environment. Test the image, installs, model download and cache +path, serving command, port, launch flags, local model request, and expected +model behavior. -Use broad scheduling requirements until a real failure justifies narrowing: +If the image, hardware choice, or major install path changes, submit another +task so the changed setup is tested before service verification. -- express GPU need as memory/count first; -- avoid pinning GPU model, backend, region, CPU, RAM, disk, or instance type from the first acceptable offer; -- require exact GPU model only for a known kernel/runtime path, unsupported architecture, benchmark target, or reproduced failure; -- check serving image/runtime compatibility with the backend driver before trusting the first run; -- set disk from model cache plus image/runtime overhead, not from a random offer; -- keep price and spot/on-demand constraints from the user/profile. +Do not move to a service after checking only GPU visibility, imports, logs, or a +health endpoint. Start the server inside the task and send a request that uses +the requested model. For a chat or reasoning model, check the response behavior +the endpoint is expected to support, such as reasoning output when that model is +supposed to expose it. -Separate three hardware concepts: +## Verify As A Service -- **minimum requirement**: what the service config should request for scheduling; -- **candidate offer**: what dstack may place now; -- **tested hardware**: what actually ran and passed verification. +Submit the service after the task has verified the recipe: image, command, port, +resources, env vars, cache mounts if used, backend/fleet choice, and model +request. -Never collapse tested hardware back into minimum requirements without evidence. +Use the service as a duplicate check of the same recipe under dstack service +runtime. The model request that worked locally in the task must also work +through the dstack service URL. -## Framework Choice - -Start with vLLM and SGLang for OpenAI-compatible LLM serving unless the model card points elsewhere. - -Prefer vLLM when: - -- vLLM recipes or docs provide a current command for the model/hardware; -- the model is standard dense/decoder-only and the goal is a straightforward OpenAI-compatible service; -- the recipe gives usable tensor-parallel, dtype, quantization, or image guidance. - -Prefer SGLang when: - -- SGLang Cookbook has a model-specific recipe; -- the model path needs SGLang-specific support, speculative decoding, radix/KV reuse, structured output performance, multimodal support, or long-context behavior; -- vLLM support is absent, degraded, or known to miss a required quantization/model path. - -Consider another server only after evidence says both vLLM and SGLang are poor fits for the requested model or API. - -For advanced serving shapes, keep scope honest. P/D disaggregation, multi-service router/worker layouts, autoscaling tuning, benchmarking loops, speculative decoding tuning, kernel changes, and production load optimization are separate work unless required to make the model serve at all. - -## Choosing The Experiment - -Submit a service directly only when these are already known: - -- pinned image or install path; -- launch command and port; -- model/API shape; -- resource envelope; -- expected health and final verification request. - -Do not use `:latest` for a final serving image unless a source or a prior run proves it is compatible with the selected backend/runtime. If a run fails because CUDA/PyTorch/vLLM requires a newer NVIDIA driver than the host provides, change the image/runtime tag or select compatible hardware; do not retry the same image. - -Use a dev environment when an interactive shell can answer a main uncertainty faster than repeated service submissions: - -- GPU runtime/image compatibility; -- import/install friction; -- model download/auth; -- launch flags; -- memory at load time; -- framework-specific error messages; -- a command that must be tuned before it becomes a service. - -Use a task when the question is one-shot: - -- `nvidia-smi`/driver/runtime sanity; -- package import or version; -- model cache/download check; -- short server start/probe; -- backend provisioning smoke test. - -Use a service when URL wiring, probes, serving process lifetime, or final API behavior is the question. - -For container-style backends that cannot pre-provision reusable instances, do not invent a pre-provisioning step. Use plans/offers and detached runs. For VM or SSH fleets, dev environments can be efficient because image/runtime work can be reused interactively. - -## Candidate Discipline - -Every candidate must have a reason to exist. Record: - -- hypothesis; -- run type; -- run name, different from any higher-level object whose logs or status are being tracked; -- config path; -- framework/image/command; -- requested resources; -- dstack plan result; -- run name/id after submission; -- current status; -- decision: keep, promote, retry with change, or stop; -- exact reason for the decision. - -Change one important variable at a time unless the previous candidate failed before testing any useful assumption. Examples of valid next changes: - -- image tag or install method; -- vLLM vs SGLang; -- dtype/quantization; -- tensor parallelism; -- GPU memory/count requirement; -- model name/path or trust-remote-code style flag when documented; -- auth/env/volume/model-cache handling; -- backend/fleet constraint after a placement or provisioning diagnosis. - -Retrying the same YAML after the same error is not prototyping. - -For endpoint deployment agents, do not name a candidate service exactly like the endpoint. Endpoint logs and service logs must stay distinguishable during failure diagnosis. Prefer short attempt names and increment them only when submitting a materially different candidate. - -## Reading dstack Evidence - -Use `/dstack` for the exact commands. Interpret evidence this way: - -- plan output: placement and price preview only; -- offers: current capacity candidates only; -- run JSON: authoritative run identity, status, service URL/model fields, jobs, and actual placed resources; -- events: lifecycle and backend/provisioning transitions; -- logs: process behavior, image/model/framework errors; -- attached shell: only for interactive diagnosis or command tuning, not for final proof. - -When waiting for a service, poll run JSON and stop waiting on terminal states. Do not wait only for log markers such as "Uvicorn running"; the process may already have failed, and the next useful action is log diagnosis. - -Use normal logs before diagnostic logs. Diagnostic logs can contain runtime environment details; use them only when normal logs and run JSON are not enough to identify whether the issue is dstack/backend infrastructure rather than the workload. - -If dstack/backend behavior appears broken, isolate it with a minimal non-application reproduction outside the repo you are working in. Do not let endpoint-agent mistakes and backend bugs blur together. - -## Failure Routing - -Classify before editing: - -- no matching offer or no fleet; -- budget/profile constraint too narrow; -- backend provisioning stalled or failed; -- image pull/runtime mismatch; -- dependency import/install failed; -- model download/gated auth/cache failed; -- server process exited; -- OOM at load or during first request; -- port/probe/service URL mismatch; -- model API works but wrong model name/API shape; -- dstack bug or backend integration bug. - -For provisioning stalls, inspect dstack run JSON and events first. Use SSH/backend-native inspection only when dstack evidence points to an instance/backend problem and the user/project context permits it. - -## Final Service Proof - -Do not treat `running`, a passed service probe, or clean logs as final success. - -For OpenAI-compatible model serving, success requires a real request to the model endpoint: - -- use the requested model name; -- include authentication only as required by dstack/service config; -- verify HTTP success; -- verify response schema; -- verify generated content exists; -- verify no model-name mismatch; -- record the URL shape, request body without secrets, response evidence, run id/name, and actual hardware from run JSON. - -If the final deliverable is a service, dev environments and tasks are evidence only. Promote the working command into a clean service and verify that service. - -## Promotion - -Promote a candidate only when the service is clean and reproducible: - -- remove debug/install/probe commands that were only for investigation; -- include applicable backend, fleet, price, spot, env-name, volume, and region constraints in the final YAML instead of relying only on apply-time flags; -- preserve the working image, command, port, model, env names, volumes, and required resource envelope; -- preserve broad minimum resources separately from actual tested hardware; -- keep source URLs and important run evidence with the final report; -- stop or mark ruled-out paid candidates unless they are still needed for active debugging. - -The final report should let another agent or developer understand: - -- why this framework and hardware envelope were chosen; -- which alternatives failed or were rejected; -- which source URLs grounded the recipe; -- the final service run id/name; -- the exact verification request/result; -- the actual hardware that passed. +If service verification fails because the image, install, model download, +command, resources, cache, or model behavior needs to change, go back to a task. +If the recipe is still right and only the service config is wrong, fix the +service config and submit the service again. diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py index ef7cec4719..51b016ecbe 100644 --- a/src/dstack/_internal/cli/commands/endpoint.py +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -1,5 +1,4 @@ import argparse -import base64 import sys import time from typing import Iterable @@ -11,6 +10,7 @@ EndpointNameCompleter, EndpointPresetNameCompleter, ) +from dstack._internal.cli.services.endpoint_logs import EndpointLogPoller from dstack._internal.cli.utils.common import ( LIVE_TABLE_PROVISION_INTERVAL_SECS, LIVE_TABLE_REFRESH_RATE_PER_SEC, @@ -27,7 +27,6 @@ from dstack._internal.cli.utils.preset import print_endpoint_presets_table from dstack._internal.core.errors import ResourceNotExistsError from dstack._internal.core.models.endpoints import Endpoint -from dstack._internal.server.schemas.logs import PollLogsRequest from dstack._internal.utils.json_utils import pydantic_orjson_dumps_with_indent @@ -50,8 +49,8 @@ def _register(self): "-a", "--all", help=( - "Show all endpoints. By default, it only shows unfinished endpoints " - "and the last finished endpoint." + "Show all endpoints. By default, it shows unfinished endpoints, " + "or the last finished endpoint if there are no unfinished endpoints." ), action="store_true", ) @@ -81,6 +80,12 @@ def _register(self): "name", help="The name of the endpoint", ).completer = EndpointNameCompleter() # type: ignore[attr-defined] + logs_parser.add_argument( + "-w", + "--watch", + help="Watch endpoint logs in realtime", + action="store_true", + ) logs_parser.add_argument( "--since", help=( @@ -125,6 +130,9 @@ def _register(self): help="Manage endpoint presets", formatter_class=self._parser.formatter_class, ) + preset_parser.add_argument( + "-v", "--verbose", action="store_true", help="Show more information" + ) preset_parser.set_defaults(subfunc=self._preset_list) preset_subparsers = preset_parser.add_subparsers(dest="preset_action") @@ -133,6 +141,13 @@ def _register(self): help="List endpoint presets", formatter_class=self._parser.formatter_class, ) + preset_list_parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=argparse.SUPPRESS, + help="Show more information", + ) preset_list_parser.set_defaults(subfunc=self._preset_list) preset_get_parser = preset_subparsers.add_parser( @@ -141,8 +156,9 @@ def _register(self): formatter_class=self._parser.formatter_class, ) preset_get_parser.add_argument( - "name", - help="The name of the endpoint preset", + "model", + metavar="MODEL", + help="The model of the endpoint preset", ).completer = EndpointPresetNameCompleter() # type: ignore[attr-defined] preset_get_parser.add_argument( "--json", @@ -157,8 +173,9 @@ def _register(self): formatter_class=self._parser.formatter_class, ) preset_delete_parser.add_argument( - "name", - help="The name of the endpoint preset", + "model", + metavar="MODEL", + help="The model of the endpoint preset", ).completer = EndpointPresetNameCompleter() # type: ignore[attr-defined] preset_delete_parser.add_argument( "-y", "--yes", help="Don't ask for confirmation", action="store_true" @@ -204,33 +221,23 @@ def _logs(self, args: argparse.Namespace): start_time = get_start_time(args.since) try: - for log in self._get_endpoint_logs(endpoint=endpoint, start_time=start_time): + for log in self._get_endpoint_logs( + endpoint=endpoint, start_time=start_time, watch=args.watch + ): sys.stdout.buffer.write(log) sys.stdout.buffer.flush() except KeyboardInterrupt: pass - def _get_endpoint_logs(self, endpoint: Endpoint, start_time) -> Iterable[bytes]: - next_token = None + def _get_endpoint_logs( + self, endpoint: Endpoint, start_time, watch: bool = False + ) -> Iterable[bytes]: + poller = EndpointLogPoller(api=self.api, endpoint=endpoint, start_time=start_time) while True: - resp = self.api.client.logs.poll( - project_name=self.api.project, - body=PollLogsRequest( - run_name=endpoint.name, - job_submission_id=endpoint.id, - start_time=start_time, - end_time=None, - descending=False, - limit=1000, - diagnose=False, - next_token=next_token, - ), - ) - for log in resp.logs: - yield base64.b64decode(log.message) - next_token = resp.next_token - if next_token is None: + yield from poller.poll() + if not watch: break + time.sleep(LIVE_TABLE_PROVISION_INTERVAL_SECS) def _stop(self, args: argparse.Namespace): try: @@ -269,7 +276,7 @@ def _get(self, args: argparse.Namespace): def _preset_list(self, args: argparse.Namespace): presets = self.api.client.endpoint_presets.list(self.api.project) - print_endpoint_presets_table(presets) + print_endpoint_presets_table(presets, verbose=args.verbose) def _preset_get(self, args: argparse.Namespace): if not args.json: @@ -278,20 +285,22 @@ def _preset_get(self, args: argparse.Namespace): try: preset = self.api.client.endpoint_presets.get( project_name=self.api.project, - name=args.name, + model=args.model, ) except ResourceNotExistsError: - console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + console.print(f"Endpoint preset for model [code]{args.model}[/] does not exist") exit(1) print(pydantic_orjson_dumps_with_indent(preset.dict(), default=None)) def _preset_delete(self, args: argparse.Namespace): presets = self.api.client.endpoint_presets.list(self.api.project) - if args.name not in {preset.name for preset in presets}: - console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + if args.model not in {preset.model for preset in presets}: + console.print(f"Endpoint preset for model [code]{args.model}[/] does not exist") exit(1) - if not args.yes and not confirm_ask(f"Delete the endpoint preset [code]{args.name}[/]?"): + if not args.yes and not confirm_ask( + f"Delete the endpoint preset for model [code]{args.model}[/]?" + ): console.print("\nExiting...") return @@ -299,10 +308,10 @@ def _preset_delete(self, args: argparse.Namespace): with console.status("Deleting endpoint preset..."): self.api.client.endpoint_presets.delete( project_name=self.api.project, - names=[args.name], + models=[args.model], ) except ResourceNotExistsError: - console.print(f"Endpoint preset [code]{args.name}[/] does not exist") + console.print(f"Endpoint preset for model [code]{args.model}[/] does not exist") exit(1) - console.print(f"Endpoint preset [code]{args.name}[/] deleted") + console.print(f"Endpoint preset for model [code]{args.model}[/] deleted") diff --git a/src/dstack/_internal/cli/services/completion.py b/src/dstack/_internal/cli/services/completion.py index ca5353c9b7..67083980af 100644 --- a/src/dstack/_internal/cli/services/completion.py +++ b/src/dstack/_internal/cli/services/completion.py @@ -77,7 +77,7 @@ def fetch_resource_names(self, api: Client) -> Iterable[str]: class EndpointPresetNameCompleter(BaseAPINameCompleter): def fetch_resource_names(self, api: Client) -> Iterable[str]: - return [r.name for r in api.client.endpoint_presets.list(api.project)] + return [r.model for r in api.client.endpoint_presets.list(api.project)] class GatewayNameCompleter(BaseAPINameCompleter): diff --git a/src/dstack/_internal/cli/services/configurators/endpoint.py b/src/dstack/_internal/cli/services/configurators/endpoint.py index d61f16268d..5368821982 100644 --- a/src/dstack/_internal/cli/services/configurators/endpoint.py +++ b/src/dstack/_internal/cli/services/configurators/endpoint.py @@ -9,6 +9,7 @@ ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator, ) +from dstack._internal.cli.services.endpoint_logs import EndpointLogPoller from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args from dstack._internal.cli.utils.common import ( NO_OFFERS_WARNING, @@ -178,9 +179,11 @@ def apply_args(self, conf: EndpointConfiguration, args: argparse.Namespace): self.apply_env_vars(conf.env, args) def _follow_endpoint_apply(self, endpoint: Endpoint): + log_poller = EndpointLogPoller(api=self.api, endpoint=endpoint) try: with MultiItemStatus(_get_apply_status(endpoint), console=console) as live: while not _is_endpoint_apply_finished(endpoint): + _print_endpoint_log_batch(log_poller.poll()) live.update( get_endpoints_table([endpoint], format_date=local_time), status=_get_apply_status(endpoint), @@ -190,6 +193,7 @@ def _follow_endpoint_apply(self, endpoint: Endpoint): project_name=self.api.project, name=endpoint.name, ) + _print_endpoint_log_batch(log_poller.poll()) except KeyboardInterrupt: console.print("\nDetached") return @@ -211,6 +215,11 @@ def _follow_endpoint_apply(self, endpoint: Endpoint): exit(1) +def _print_endpoint_log_batch(logs: list[bytes]) -> None: + for log in logs: + console.out(log.decode(errors="replace"), end="") + + def _apply_profile(conf: EndpointConfiguration, profile: Profile): for field in ProfileParams.__fields__: value = getattr(profile, field) @@ -258,12 +267,8 @@ def th(s: str) -> str: props.add_row(th("Max price"), _format_max_price(plan)) props.add_row(th("Preset policy"), plan.preset_policy.value) if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): - props.add_row(th("Preset"), plan.provisioning_plan.preset_name) - elif ( - isinstance(plan.provisioning_plan, EndpointProvisioningPlanAgent) - and plan.provisioning_plan.max_budget is not None - ): - props.add_row(th("Agent budget"), _format_price_limit(plan.provisioning_plan.max_budget)) + props.add_row(th("Preset"), plan.provisioning_plan.preset_model) + props.add_row(th("Recipe"), plan.provisioning_plan.recipe_id) console.print(props) console.print() diff --git a/src/dstack/_internal/cli/services/endpoint_logs.py b/src/dstack/_internal/cli/services/endpoint_logs.py new file mode 100644 index 0000000000..d629c84133 --- /dev/null +++ b/src/dstack/_internal/cli/services/endpoint_logs.py @@ -0,0 +1,76 @@ +import base64 +from collections import Counter +from datetime import datetime, timedelta +from typing import Optional + +from dstack._internal.core.models.endpoints import Endpoint +from dstack._internal.core.models.logs import LogEvent +from dstack._internal.server.schemas.logs import PollLogsRequest + +_ENDPOINT_LOG_WATCH_OVERLAP = timedelta(seconds=20) + + +class EndpointLogPoller: + def __init__(self, *, api, endpoint: Endpoint, start_time: Optional[datetime] = None) -> None: + self._api = api + self._endpoint = endpoint + self._next_start_time = start_time + self._latest_printed_at: Optional[datetime] = None + self._seen_log_counts: Counter[tuple[datetime, str, str]] = Counter() + + def poll(self) -> list[bytes]: + logs: list[bytes] = [] + next_token = None + batch_log_counts: Counter[tuple[datetime, str, str]] = Counter() + while True: + resp = self._api.client.logs.poll( + project_name=self._api.project, + body=PollLogsRequest( + run_name=self._endpoint.name, + job_submission_id=self._endpoint.id, + start_time=self._next_start_time, + end_time=None, + descending=False, + limit=1000, + diagnose=False, + next_token=next_token, + ), + ) + for log in resp.logs: + if not self._should_print(log, batch_log_counts=batch_log_counts): + continue + logs.append(_format_log(log)) + next_token = resp.next_token + if next_token is None: + break + if self._latest_printed_at is not None: + self._next_start_time = self._latest_printed_at - _ENDPOINT_LOG_WATCH_OVERLAP + self._cleanup_seen_log_counts(before=self._next_start_time) + return logs + + def _should_print( + self, log: LogEvent, *, batch_log_counts: Counter[tuple[datetime, str, str]] + ) -> bool: + key = _log_key(log) + batch_log_counts[key] += 1 + if batch_log_counts[key] <= self._seen_log_counts[key]: + return False + self._seen_log_counts[key] += 1 + if self._latest_printed_at is None or log.timestamp > self._latest_printed_at: + self._latest_printed_at = log.timestamp + return True + + def _cleanup_seen_log_counts(self, before: datetime) -> None: + for key in list(self._seen_log_counts): + timestamp, _, _ = key + if timestamp <= before: + del self._seen_log_counts[key] + + +def _log_key(log: LogEvent) -> tuple[datetime, str, str]: + return (log.timestamp, log.log_source.value, log.message) + + +def _format_log(log: LogEvent) -> bytes: + timestamp = log.timestamp.astimezone().strftime("%Y-%m-%d %H:%M:%S") + return f"[{timestamp}] ".encode() + base64.b64decode(log.message) diff --git a/src/dstack/_internal/cli/utils/endpoint.py b/src/dstack/_internal/cli/utils/endpoint.py index a61e737230..5dd740d613 100644 --- a/src/dstack/_internal/cli/utils/endpoint.py +++ b/src/dstack/_internal/cli/utils/endpoint.py @@ -11,7 +11,6 @@ def filter_endpoints_for_listing( endpoints: List[Endpoint], show_all: bool = False, limit: int | None = None, - include_latest_finished: bool = True, ) -> List[Endpoint]: endpoints = sorted( endpoints, @@ -23,18 +22,10 @@ def filter_endpoints_for_listing( if show_all: return endpoints - latest_finished = None - filtered = [] - for endpoint in endpoints: - if _is_endpoint_finished(endpoint): - if not include_latest_finished: - continue - if latest_finished is None: - latest_finished = endpoint - filtered.append(endpoint) - continue - filtered.append(endpoint) - return filtered + unfinished = [endpoint for endpoint in endpoints if not _is_endpoint_finished(endpoint)] + if unfinished: + return unfinished + return endpoints[:1] def print_endpoints_table(endpoints: List[Endpoint], verbose: bool = False): @@ -70,7 +61,8 @@ def th(value: str) -> str: endpoint.status_message, ), ) - table.add_row(th("Run"), endpoint.run_name or "-") + table.add_row(th("Preset policy"), endpoint.configuration.preset_policy.value) + table.add_row(th("Service run"), endpoint.run_name or "-") table.add_row(th("URL"), endpoint.url or "-") table.add_row(th("Created"), format_date(endpoint.created_at)) if endpoint.status_message: @@ -87,8 +79,9 @@ def get_endpoints_table( table.add_column("NAME", no_wrap=True) table.add_column("MODEL") table.add_column("STATUS", no_wrap=True) - table.add_column("RUN") + table.add_column("POLICY", no_wrap=True) if verbose: + table.add_column("SERVICE RUN") table.add_column("URL") table.add_column("CREATED") if verbose: @@ -102,7 +95,8 @@ def get_endpoints_table( endpoint.status, endpoint.status_message, ), - "RUN": endpoint.run_name or "-", + "POLICY": endpoint.configuration.preset_policy.value, + "SERVICE RUN": endpoint.run_name or "-", "URL": endpoint.url or "-", "CREATED": format_date(endpoint.created_at), "ERROR": endpoint.status_message, @@ -119,7 +113,7 @@ def _format_endpoint_status( color_map = { EndpointStatus.SUBMITTED: "grey", EndpointStatus.PROVISIONING: "deep_sky_blue1", - EndpointStatus.CLAUDING: "medium_purple1", + EndpointStatus.PROTOTYPING: "medium_purple1", EndpointStatus.RUNNING: "sea_green3", EndpointStatus.STOPPING: "deep_sky_blue1", EndpointStatus.STOPPED: "grey62", diff --git a/src/dstack/_internal/cli/utils/preset.py b/src/dstack/_internal/cli/utils/preset.py index e09c095b42..a5b066c6c9 100644 --- a/src/dstack/_internal/cli/utils/preset.py +++ b/src/dstack/_internal/cli/utils/preset.py @@ -3,45 +3,78 @@ from rich.table import Table from dstack._internal.cli.utils.common import add_row_from_dict, console -from dstack._internal.core.models.endpoint_presets import EndpointPreset +from dstack._internal.core.models.endpoint_presets import EndpointPreset, EndpointPresetRecipe +from dstack._internal.utils.common import pretty_resources -def print_endpoint_presets_table(presets: List[EndpointPreset]): - table = get_endpoint_presets_table(presets) +def print_endpoint_presets_table(presets: List[EndpointPreset], verbose: bool = False): + table = get_endpoint_presets_table(presets, verbose=verbose) console.print(table) console.print() -def get_endpoint_presets_table(presets: List[EndpointPreset]) -> Table: +def get_endpoint_presets_table(presets: List[EndpointPreset], verbose: bool = False) -> Table: table = Table(box=None) - table.add_column("NAME", style="bold", no_wrap=True) - table.add_column("MODEL") - table.add_column("RESOURCES") + table.add_column("MODEL", no_wrap=True) + if verbose: + table.add_column("RESOURCES") + else: + table.add_column("GPU") for preset in presets: - if len(preset.replica_spec_groups) == 1: - add_row_from_dict( + if len(preset.recipes) == 1: + _add_recipe_rows( table, - { - "NAME": preset.name, - "MODEL": preset.model, - "RESOURCES": _format_resources(preset.replica_spec_groups[0].resources), - }, + label=f"[bold]{preset.model}[/]", + recipe=preset.recipes[0], + verbose=verbose, ) continue - add_row_from_dict(table, {"NAME": preset.name, "MODEL": preset.model}) - for group in preset.replica_spec_groups: - add_row_from_dict( + add_row_from_dict(table, {"MODEL": f"[bold]{preset.model}[/]"}) + for recipe_num, recipe in enumerate(preset.recipes): + _add_recipe_rows( table, - { - "NAME": f" group={group.name}", - "RESOURCES": _format_resources(group.resources), - }, + label=f"[secondary] recipe={recipe_num}[/]", + recipe=recipe, + verbose=verbose, ) return table -def _format_resources(resources) -> str: +def _add_recipe_rows( + table: Table, + label: str, + recipe: EndpointPresetRecipe, + verbose: bool, +) -> None: + groups = recipe.service.replica_groups + column = "RESOURCES" if verbose else "GPU" + if len(groups) == 1: + add_row_from_dict( + table, + { + "MODEL": label, + column: _format_resources(groups[0].resources, verbose=verbose), + }, + ) + return + + add_row_from_dict(table, {"MODEL": label}) + for group in groups: + add_row_from_dict( + table, + { + "MODEL": f"[secondary] group={group.name}[/]", + column: _format_resources(group.resources, verbose=verbose), + }, + ) + + +def _format_resources(resources, verbose: bool) -> str: + if not verbose: + if resources.gpu is None: + return "-" + return _format_gpu(resources) formatted = resources.pretty_format() if resources.gpu is not None and resources.gpu.count.min == 0: if resources.gpu.count.max in (None, 0): @@ -52,3 +85,19 @@ def _format_resources(resources) -> str: .replace("gpu=0", "-") ) return formatted + + +def _format_gpu(resources) -> str: + gpu = resources.gpu + assert gpu is not None + if gpu.count.min == 0 and gpu.count.max in (None, 0): + return "-" + formatted = pretty_resources( + gpu_vendor=gpu.vendor, + gpu_name=",".join(gpu.name) if gpu.name else None, + gpu_count=gpu.count, + gpu_memory=gpu.memory, + total_gpu_memory=gpu.total_memory, + compute_capability=gpu.compute_capability, + ) + return formatted.removeprefix("gpu=") or "-" diff --git a/src/dstack/_internal/core/models/endpoint_presets.py b/src/dstack/_internal/core/models/endpoint_presets.py index 78a4403397..1e8b7db79b 100644 --- a/src/dstack/_internal/core/models/endpoint_presets.py +++ b/src/dstack/_internal/core/models/endpoint_presets.py @@ -3,21 +3,22 @@ from dstack._internal.core.models.resources import ResourcesSpec -class EndpointPresetReplicaSpecGroup(CoreModel): - """Ordered to match service replica groups; "0" is the implicit group.""" +class EndpointPresetValidationReplica(CoreModel): + resources: list[ResourcesSpec] + """Exact resources for each running replica in this service replica group.""" - name: str - resources: ResourcesSpec - """Per-replica scheduling requirements used when applying the preset.""" - tested_resources: list[ResourcesSpec] - """Exact resources of the replicas that were running when the preset was verified.""" +class EndpointPresetValidation(CoreModel): + replicas: list[EndpointPresetValidationReplica] + """Ordered to match `ServiceConfiguration.replica_groups`.""" -class EndpointPreset(CoreModel): - name: str - model: str - replica_spec_groups: list[EndpointPresetReplicaSpecGroup] - -class EndpointPresetDetails(EndpointPreset): +class EndpointPresetRecipe(CoreModel): + id: str service: ServiceConfiguration + validations: list[EndpointPresetValidation] + + +class EndpointPreset(CoreModel): + model: str + recipes: list[EndpointPresetRecipe] diff --git a/src/dstack/_internal/core/models/endpoints.py b/src/dstack/_internal/core/models/endpoints.py index 15ac989b0a..032b96921c 100644 --- a/src/dstack/_internal/core/models/endpoints.py +++ b/src/dstack/_internal/core/models/endpoints.py @@ -19,7 +19,7 @@ class EndpointStatus(str, Enum): SUBMITTED = "submitted" PROVISIONING = "provisioning" - CLAUDING = "clauding" + PROTOTYPING = "prototyping" RUNNING = "running" STOPPING = "stopping" STOPPED = "stopped" @@ -62,6 +62,9 @@ class EndpointConfiguration( Env, Field(description="The mapping or the list of environment variables"), ] = Env() + # TODO: Add endpoint-level resources only with hard scheduling semantics for both + # single-service and replica-group presets. V1 intentionally relies on preset/service + # resources plus ProfileParams constraints. preset_policy: Annotated[ EndpointPresetPolicy, Field( @@ -72,16 +75,6 @@ class EndpointConfiguration( ) ), ] = EndpointPresetPolicy.REUSE_OR_CREATE - max_agent_budget: Annotated[ - Optional[float], - Field( - description=( - "The maximum agent spend for provisioning this endpoint, in dollars. " - "If not specified, the server default is used." - ), - gt=0.0, - ), - ] = None class Endpoint(CoreModel): @@ -104,16 +97,6 @@ class EndpointProvisioningPlanNone(CoreModel): reason: str -class EndpointPlanReplicaSpecGroup(CoreModel): - """Ordered to match service replica groups; "0" is the implicit group.""" - - name: str - resources: ResourcesSpec - """Per-replica scheduling requirements used for offer matching.""" - tested_resources: list[ResourcesSpec] - """Exact resources of the replicas that were running when the preset was verified.""" - - class EndpointPlanJobOffers(CoreModel): replica_group: str resources: ResourcesSpec @@ -126,16 +109,15 @@ class EndpointPlanJobOffers(CoreModel): class EndpointProvisioningPlanPreset(CoreModel): type: Literal["preset"] = "preset" - preset_name: str + preset_model: str + recipe_id: str service_name: str - replica_spec_groups: list[EndpointPlanReplicaSpecGroup] job_offers: list[EndpointPlanJobOffers] class EndpointProvisioningPlanAgent(CoreModel): type: Literal["agent"] = "agent" agent_model: str - max_budget: Optional[float] = None reason: Optional[str] = None diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py index 64082c1db1..39bc45768d 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -45,9 +45,12 @@ emit_endpoint_status_change_event, get_endpoint_agent_admin_required_message, get_endpoint_configuration, + get_endpoint_no_fleets_message, + has_endpoint_existing_usable_fleets, record_endpoint_run_submission, ) from dstack._internal.server.services.endpoints.agent import ( + abort_agent_endpoint, get_agent_service, get_agent_unavailable_reason, ) @@ -166,7 +169,7 @@ async def fetch(self, limit: int) -> list[EndpointPipelineItem]: [ EndpointStatus.SUBMITTED, EndpointStatus.PROVISIONING, - EndpointStatus.CLAUDING, + EndpointStatus.PROTOTYPING, EndpointStatus.RUNNING, EndpointStatus.STOPPING, ] @@ -249,8 +252,8 @@ async def process(self, item: EndpointPipelineItem): endpoint_model=endpoint_model, pipeline_hinter=self._pipeline_hinter, ) - elif endpoint_model.status == EndpointStatus.CLAUDING: - result = await _process_clauding_endpoint( + elif endpoint_model.status == EndpointStatus.PROTOTYPING: + result = await _process_prototyping_endpoint( endpoint_model=endpoint_model, pipeline_hinter=self._pipeline_hinter, ) @@ -394,13 +397,14 @@ class _ProcessResult: @dataclass(frozen=True) class _PresetSubmission: run_id: uuid.UUID - preset_name: str + preset_model: str + recipe_id: str @dataclass(frozen=True) class _PresetSubmissionResult: submission: Optional[_PresetSubmission] = None - unprovisionable_preset_name: Optional[str] = None + unprovisionable_preset: Optional[str] = None async def _get_endpoint_runs_to_stop_after_failure( @@ -442,6 +446,20 @@ async def _process_submitted_endpoint( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: + endpoint_configuration = get_endpoint_configuration(endpoint_model) + async with get_session_ctx() as session: + has_usable_fleets = await has_endpoint_existing_usable_fleets( + session=session, + project=endpoint_model.project, + configuration=endpoint_configuration, + ) + if not has_usable_fleets: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": get_endpoint_no_fleets_message(endpoint_configuration), + } + ) try: submission_result = await _submit_endpoint_from_preset( endpoint_id=endpoint_model.id, @@ -456,22 +474,26 @@ async def _process_submitted_endpoint( ) if submission_result.submission is not None: logger.info( - "Provisioning endpoint %s from preset %s", + "Provisioning endpoint %s from preset %s recipe %s", endpoint_model.name, - submission_result.submission.preset_name, + submission_result.submission.preset_model, + submission_result.submission.recipe_id, ) update_map = _EndpointUpdateMap( status=EndpointStatus.PROVISIONING, status_message=None, service_run_id=submission_result.submission.run_id, - provisioning_method=f"preset:{submission_result.submission.preset_name}", + provisioning_method=( + f"preset:{submission_result.submission.preset_model}" + f"#{submission_result.submission.recipe_id}" + ), ) return _ProcessResult(update_map=update_map) - if _should_provision_with_agent(endpoint_model): + if await _should_provision_with_agent(endpoint_model): logger.info("Provisioning endpoint %s with server agent", endpoint_model.name) update_map = _EndpointUpdateMap( - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, status_message=None, provisioning_method="agent", ) @@ -483,19 +505,26 @@ async def _process_submitted_endpoint( "status": EndpointStatus.FAILED, "status_message": _get_no_provisioning_path_message( endpoint_model, - unprovisionable_preset_name=submission_result.unprovisionable_preset_name, + unprovisionable_preset=submission_result.unprovisionable_preset, ), } ) -def _should_provision_with_agent(endpoint_model: EndpointModel) -> bool: +async def _should_provision_with_agent(endpoint_model: EndpointModel) -> bool: endpoint_configuration = get_endpoint_configuration(endpoint_model) - return ( - endpoint_configuration.preset_policy != EndpointPresetPolicy.REUSE - and get_agent_service().is_enabled() - and can_use_endpoint_agent(user=endpoint_model.user, project=endpoint_model.project) - ) + if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE: + return False + if not get_agent_service().is_enabled(): + return False + if not can_use_endpoint_agent(user=endpoint_model.user, project=endpoint_model.project): + return False + async with get_session_ctx() as session: + return await has_endpoint_existing_usable_fleets( + session=session, + project=endpoint_model.project, + configuration=endpoint_configuration, + ) async def _get_active_serving_run_name_conflict( @@ -529,7 +558,7 @@ async def _process_provisioning_endpoint( ) -> _ProcessResult: if endpoint_model.service_run is None: if endpoint_model.provisioning_method == "agent": - return await _process_clauding_endpoint( + return await _process_prototyping_endpoint( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) @@ -548,12 +577,17 @@ async def _process_provisioning_endpoint( if readiness.model_base_url is None or readiness.model_name is None: return _ProcessResult() if endpoint_model.provisioning_method == "agent": - # The agent's verification report is the functional signal. This server-side gate only + # The agent's final report is the functional signal. This server-side gate only # confirms that the verified run still looks like a normal ready dstack service. await _try_save_agent_endpoint_preset( endpoint_model=endpoint_model, - model_name=readiness.model_name, ) + if endpoint_model.service_run_id is not None: + await _stop_non_final_submitted_runs( + endpoint_model=endpoint_model, + final_run_id=endpoint_model.service_run_id, + pipeline_hinter=pipeline_hinter, + ) return _ProcessResult( update_map={ "status": EndpointStatus.RUNNING, @@ -562,19 +596,25 @@ async def _process_provisioning_endpoint( ) -async def _process_clauding_endpoint( +async def _process_prototyping_endpoint( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: if endpoint_model.service_run is not None: - return await _process_agent_verified_endpoint(endpoint_model) + return await _process_agent_verified_endpoint( + endpoint_model=endpoint_model, + pipeline_hinter=pipeline_hinter, + ) return await _provision_endpoint_with_agent( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) -async def _process_agent_verified_endpoint(endpoint_model: EndpointModel) -> _ProcessResult: +async def _process_agent_verified_endpoint( + endpoint_model: EndpointModel, + pipeline_hinter: PipelineHinterProtocol, +) -> _ProcessResult: run_model = endpoint_model.service_run if run_model is None: return _ProcessResult() @@ -587,11 +627,15 @@ async def _process_agent_verified_endpoint(endpoint_model: EndpointModel) -> _Pr } ) if readiness.model_base_url is None or readiness.model_name is None: - return _ProcessResult(update_map={"status": EndpointStatus.CLAUDING}) + return _ProcessResult(update_map={"status": EndpointStatus.PROTOTYPING}) await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, - model_name=readiness.model_name, + ) + await _stop_non_final_submitted_runs( + endpoint_model=endpoint_model, + final_run_id=run_model.id, + pipeline_hinter=pipeline_hinter, ) return _ProcessResult( update_map={ @@ -618,14 +662,15 @@ async def _provision_endpoint_with_agent( endpoint_model=endpoint_model, pipeline_hinter=pipeline_hinter, ) - await _record_agent_candidate_run_submissions( + await _record_agent_submitted_runs( endpoint_model=endpoint_model, - candidate_run_ids=result.candidate_run_ids, + submitted_run_ids=result.submitted_run_ids, + submitted_run_names=result.submitted_run_names, ) if result.in_progress: return _ProcessResult( update_map={ - "status": EndpointStatus.CLAUDING, + "status": EndpointStatus.PROTOTYPING, "status_message": None, } ) @@ -641,7 +686,7 @@ async def _provision_endpoint_with_agent( return _ProcessResult( update_map={ "status": EndpointStatus.FAILED, - "status_message": "Server agent did not return a verification report", + "status_message": "Server agent did not return a final report", } ) if not report.success: @@ -659,7 +704,7 @@ async def _provision_endpoint_with_agent( return _ProcessResult( update_map={ "status": EndpointStatus.FAILED, - "status_message": "Server agent returned inconsistent run id in verification report", + "status_message": "Server agent returned inconsistent run id in final report", } ) if ( @@ -670,16 +715,14 @@ async def _provision_endpoint_with_agent( return _ProcessResult( update_map={ "status": EndpointStatus.FAILED, - "status_message": ( - "Server agent returned inconsistent run name in verification report" - ), + "status_message": ("Server agent returned inconsistent run name in final report"), } ) if run_id is None: return _ProcessResult( update_map={ "status": EndpointStatus.FAILED, - "status_message": "Server agent verification report did not identify a run id", + "status_message": "Server agent final report did not identify a run id", } ) @@ -702,6 +745,23 @@ async def _provision_endpoint_with_agent( "status_message": (f"Server agent reported run '{run_id}' but it was not found"), } ) + if report.run_name is not None and report.run_name != run_model.run_name: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": ("Server agent returned inconsistent run name in final report"), + } + ) + if not _is_valid_agent_submission_run_name(endpoint_model.name, run_model.run_name): + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": ( + "Server agent final service run name must be " + f"'{endpoint_model.name}-'" + ), + } + ) if run_model.user_id != endpoint_model.user_id: return _ProcessResult( update_map={ @@ -748,7 +808,7 @@ async def _provision_endpoint_with_agent( if readiness.model_base_url is None or readiness.model_name is None: return _ProcessResult( update_map={ - "status": EndpointStatus.CLAUDING, + "status": EndpointStatus.PROTOTYPING, "status_message": None, "service_run_id": run_model.id, } @@ -756,7 +816,11 @@ async def _provision_endpoint_with_agent( await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, - model_name=readiness.model_name, + ) + await _stop_non_final_submitted_runs( + endpoint_model=endpoint_model, + final_run_id=run_model.id, + pipeline_hinter=pipeline_hinter, ) return _ProcessResult( update_map={ @@ -789,27 +853,37 @@ async def _record_endpoint_run_submission(endpoint_id: uuid.UUID, run_id: uuid.U await session.commit() -async def _record_agent_candidate_run_submissions( +async def _record_agent_submitted_runs( *, endpoint_model: EndpointModel, - candidate_run_ids: Sequence[uuid.UUID], + submitted_run_ids: Sequence[uuid.UUID], + submitted_run_names: Sequence[str], ) -> None: - if len(candidate_run_ids) == 0: + valid_submitted_run_names = [ + run_name + for run_name in submitted_run_names + if _is_valid_agent_submission_run_name(endpoint_model.name, run_name) + ] + if len(submitted_run_ids) == 0 and len(valid_submitted_run_names) == 0: return async with get_session_ctx() as session: res = await session.execute( select(RunModel).where( - RunModel.id.in_(candidate_run_ids), + or_( + RunModel.id.in_(submitted_run_ids), + RunModel.run_name.in_(valid_submitted_run_names), + ), RunModel.project_id == endpoint_model.project_id, RunModel.user_id == endpoint_model.user_id, RunModel.deleted == False, ) ) runs_by_id = {run.id: run for run in res.scalars().all()} - for run_id in candidate_run_ids: + runs_by_name = {run.run_name: run for run in runs_by_id.values()} + for run_id in submitted_run_ids: if run_id not in runs_by_id: logger.info( - "Ignoring endpoint %s candidate run %s because it is not a live run " + "Ignoring endpoint %s submitted run %s because it is not a live run " "owned by the endpoint user/project", endpoint_model.name, run_id, @@ -823,14 +897,64 @@ async def _record_agent_candidate_run_submissions( ) except ServerClientError as e: logger.info( - "Ignoring endpoint %s candidate run %s: %s", + "Ignoring endpoint %s submitted run %s: %s", endpoint_model.name, run_id, e.msg, ) + for run_name in valid_submitted_run_names: + run = runs_by_name.get(run_name) + if run is None: + continue + try: + await record_endpoint_run_submission( + session=session, + endpoint_id=endpoint_model.id, + run_id=run.id, + ) + except ServerClientError as e: + logger.info( + "Ignoring endpoint %s submitted run %s: %s", + endpoint_model.name, + run_name, + e.msg, + ) await session.commit() +async def _stop_non_final_submitted_runs( + *, + endpoint_model: EndpointModel, + final_run_id: uuid.UUID, + pipeline_hinter: PipelineHinterProtocol, +) -> None: + for run_model in await _get_endpoint_unfinished_runs(endpoint_model): + if run_model.id == final_run_id or run_model.status == RunStatus.TERMINATING: + continue + logger.info( + "Stopping non-final endpoint run %s after endpoint %s verified run %s", + run_model.run_name, + endpoint_model.name, + final_run_id, + ) + await _stop_backing_run( + endpoint_model=endpoint_model, + run_name=run_model.run_name, + pipeline_hinter=pipeline_hinter, + ) + + +def _is_valid_agent_submission_run_name(endpoint_name: str, run_name: str) -> bool: + prefix = f"{endpoint_name}-" + if not run_name.startswith(prefix): + return False + suffix = run_name[len(prefix) :] + if not suffix.isdecimal(): + return False + submission_num = int(suffix) + return submission_num > 0 and str(submission_num) == suffix + + async def _process_running_endpoint(endpoint_model: EndpointModel) -> _ProcessResult: readiness = _get_backing_service_readiness(endpoint_model) if readiness.failed_message is not None: @@ -849,10 +973,13 @@ async def _process_stopping_endpoint( endpoint_model: EndpointModel, pipeline_hinter: PipelineHinterProtocol, ) -> _ProcessResult: - # TODO: When the Claude agent service exposes cancellation, interrupt a live - # agent process here instead of waiting for it to notice/finish. + agent_aborted = True + if endpoint_model.provisioning_method == "agent": + agent_aborted = await abort_agent_endpoint(endpoint_model) run_models = await _get_endpoint_unfinished_runs(endpoint_model) if not run_models: + if not agent_aborted: + return _ProcessResult() return _get_stopped_result() for run_model in run_models: if run_model.status == RunStatus.TERMINATING: @@ -872,7 +999,6 @@ async def _process_stopping_endpoint( async def _try_save_agent_endpoint_preset( endpoint_model: EndpointModel, - model_name: str, ) -> None: run_model = endpoint_model.service_run if run_model is None: @@ -880,18 +1006,15 @@ async def _try_save_agent_endpoint_preset( await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, - model_name=model_name, ) async def _save_agent_endpoint_preset( endpoint_model: EndpointModel, run_model: RunModel, - model_name: str, ) -> None: - preset_name = f"{model_name}-{str(run_model.id)[:8]}" try: - preset = build_endpoint_preset_from_run(name=preset_name, run_model=run_model) + preset = build_endpoint_preset_from_run(run_model) saved_preset = await get_endpoint_preset_service().save_preset( endpoint_model.project.name, preset, @@ -910,7 +1033,13 @@ async def _save_agent_endpoint_preset( exc_info=True, ) return - logger.info("Saved endpoint preset %s for endpoint %s", saved_preset.name, endpoint_model.name) + recipe_ids = ", ".join(recipe.id for recipe in saved_preset.recipes) + logger.info( + "Saved endpoint preset for model %s recipes %s for endpoint %s", + saved_preset.model, + recipe_ids, + endpoint_model.name, + ) @dataclass(frozen=True) @@ -990,10 +1119,12 @@ async def _submit_endpoint_from_preset( ) preset_plan = preset_planning_result.provisionable if preset_plan is None: - unprovisionable_preset_name = None + unprovisionable_preset = None if preset_planning_result.unprovisionable is not None: - unprovisionable_preset_name = preset_planning_result.unprovisionable.preset.name - return _PresetSubmissionResult(unprovisionable_preset_name=unprovisionable_preset_name) + unprovisionable_preset = _format_preset_plan_label( + preset_planning_result.unprovisionable + ) + return _PresetSubmissionResult(unprovisionable_preset=unprovisionable_preset) conflict_message = await _get_active_run_name_conflict_message( session=session, project=endpoint_model.project, @@ -1020,7 +1151,11 @@ async def _submit_endpoint_from_preset( ) await session.commit() return _PresetSubmissionResult( - submission=_PresetSubmission(run_id=run.id, preset_name=preset_plan.preset.name) + submission=_PresetSubmission( + run_id=run.id, + preset_model=preset_plan.preset.model, + recipe_id=preset_plan.recipe.id, + ) ) @@ -1049,13 +1184,11 @@ async def _get_active_run_name_conflict_message( def _get_no_provisioning_path_message( endpoint_model: EndpointModel, - unprovisionable_preset_name: Optional[str] = None, + unprovisionable_preset: Optional[str] = None, ) -> str: endpoint_configuration = get_endpoint_configuration(endpoint_model) - if unprovisionable_preset_name is not None: - reason = ( - f"Endpoint preset {unprovisionable_preset_name} matched but has no available offers." - ) + if unprovisionable_preset is not None: + reason = f"Endpoint preset {unprovisionable_preset} matched but has no available offers." if endpoint_configuration.preset_policy == EndpointPresetPolicy.REUSE: return reason if get_agent_service().is_enabled() and not can_use_endpoint_agent( @@ -1090,6 +1223,10 @@ def _get_no_provisioning_path_message( return f"Preset policy create requires the server agent, but {agent_unavailable_reason}" +def _format_preset_plan_label(preset_plan) -> str: + return f"for model {preset_plan.preset.model} recipe {preset_plan.recipe.id}" + + async def _stop_backing_run( endpoint_model: EndpointModel, run_name: str, diff --git a/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py b/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_sessions.py similarity index 67% rename from src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py rename to src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_sessions.py index 907d4058e3..fee7c5799e 100644 --- a/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_attempts.py +++ b/src/dstack/_internal/server/migrations/versions/2026/07_06_1700_4e1d6c2a9b77_add_endpoint_agent_sessions.py @@ -1,4 +1,4 @@ -"""add endpoint agent attempts +"""add endpoint agent sessions Revision ID: 4e1d6c2a9b77 Revises: 03efb71a1563 @@ -22,11 +22,11 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( - "endpoint_agent_attempts", + "endpoint_agent_sessions", sa.Column( "endpoint_id", sqlalchemy_utils.types.uuid.UUIDType(binary=False), nullable=False ), - sa.Column("attempt_num", sa.Integer(), nullable=False), + sa.Column("session_num", sa.Integer(), nullable=False), sa.Column("status", sa.String(length=100), nullable=False), sa.Column("workspace_path", sa.Text(), nullable=False), sa.Column("pid", sa.Integer(), nullable=True), @@ -34,8 +34,6 @@ def upgrade() -> None: sa.Column("progress_log_offset", sa.BigInteger(), nullable=False), sa.Column("stdout_log_offset", sa.BigInteger(), nullable=False), sa.Column("stderr_log_offset", sa.BigInteger(), nullable=False), - sa.Column("max_agent_budget", sa.Float(), nullable=True), - sa.Column("spent_agent_budget", sa.Float(), nullable=True), sa.Column("status_message", sa.Text(), nullable=True), sa.Column("created_at", dstack._internal.server.models.NaiveDateTime(), nullable=False), sa.Column("updated_at", dstack._internal.server.models.NaiveDateTime(), nullable=False), @@ -43,24 +41,24 @@ def upgrade() -> None: sa.ForeignKeyConstraint( ["endpoint_id"], ["endpoints.id"], - name=op.f("fk_endpoint_agent_attempts_endpoint_id_endpoints"), + name=op.f("fk_endpoint_agent_sessions_endpoint_id_endpoints"), ondelete="CASCADE", ), sa.PrimaryKeyConstraint( - "endpoint_id", "attempt_num", name=op.f("pk_endpoint_agent_attempts") + "endpoint_id", "session_num", name=op.f("pk_endpoint_agent_sessions") ), ) - with op.batch_alter_table("endpoint_agent_attempts", schema=None) as batch_op: + with op.batch_alter_table("endpoint_agent_sessions", schema=None) as batch_op: batch_op.create_index( - batch_op.f("ix_endpoint_agent_attempts_endpoint_id"), ["endpoint_id"], unique=False + batch_op.f("ix_endpoint_agent_sessions_endpoint_id"), ["endpoint_id"], unique=False ) batch_op.create_index( - "ix_endpoint_agent_attempts_endpoint_status", + "ix_endpoint_agent_sessions_endpoint_status", ["endpoint_id", "status"], unique=False, ) batch_op.create_index( - batch_op.f("ix_endpoint_agent_attempts_status"), ["status"], unique=False + batch_op.f("ix_endpoint_agent_sessions_status"), ["status"], unique=False ) # ### end Alembic commands ### @@ -68,10 +66,10 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table("endpoint_agent_attempts", schema=None) as batch_op: - batch_op.drop_index(batch_op.f("ix_endpoint_agent_attempts_status")) - batch_op.drop_index("ix_endpoint_agent_attempts_endpoint_status") - batch_op.drop_index(batch_op.f("ix_endpoint_agent_attempts_endpoint_id")) + with op.batch_alter_table("endpoint_agent_sessions", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_endpoint_agent_sessions_status")) + batch_op.drop_index("ix_endpoint_agent_sessions_endpoint_status") + batch_op.drop_index(batch_op.f("ix_endpoint_agent_sessions_endpoint_id")) - op.drop_table("endpoint_agent_attempts") + op.drop_table("endpoint_agent_sessions") # ### end Alembic commands ### diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index ec9770411f..0580abace4 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -55,7 +55,7 @@ CASCADE_DEFAULT_WITH_DELETE_ORPHAN = "save-update, merge, delete-orphan, delete" -class EndpointAgentAttemptStatus(enum.Enum): +class EndpointAgentSessionStatus(enum.Enum): RUNNING = "running" SUCCEEDED = "succeeded" FAILED = "failed" @@ -1067,17 +1067,17 @@ class EndpointRunSubmissionModel(BaseModel): ) -class EndpointAgentAttemptModel(BaseModel): - __tablename__ = "endpoint_agent_attempts" +class EndpointAgentSessionModel(BaseModel): + __tablename__ = "endpoint_agent_sessions" endpoint_id: Mapped[uuid.UUID] = mapped_column( ForeignKey("endpoints.id", ondelete="CASCADE"), primary_key=True ) endpoint: Mapped["EndpointModel"] = relationship() - attempt_num: Mapped[int] = mapped_column(Integer, primary_key=True) - status: Mapped[EndpointAgentAttemptStatus] = mapped_column( - EnumAsString(EndpointAgentAttemptStatus, 100), index=True + session_num: Mapped[int] = mapped_column(Integer, primary_key=True) + status: Mapped[EndpointAgentSessionStatus] = mapped_column( + EnumAsString(EndpointAgentSessionStatus, 100), index=True ) workspace_path: Mapped[str] = mapped_column(Text) @@ -1088,8 +1088,6 @@ class EndpointAgentAttemptModel(BaseModel): stdout_log_offset: Mapped[int] = mapped_column(BigInteger, default=0) stderr_log_offset: Mapped[int] = mapped_column(BigInteger, default=0) - max_agent_budget: Mapped[Optional[float]] = mapped_column(Float) - spent_agent_budget: Mapped[Optional[float]] = mapped_column(Float) status_message: Mapped[Optional[str]] = mapped_column(Text) created_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime) @@ -1097,8 +1095,8 @@ class EndpointAgentAttemptModel(BaseModel): finished_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) __table_args__ = ( - Index("ix_endpoint_agent_attempts_endpoint_id", endpoint_id), - Index("ix_endpoint_agent_attempts_endpoint_status", endpoint_id, status), + Index("ix_endpoint_agent_sessions_endpoint_id", endpoint_id), + Index("ix_endpoint_agent_sessions_endpoint_status", endpoint_id, status), ) diff --git a/src/dstack/_internal/server/routers/endpoints.py b/src/dstack/_internal/server/routers/endpoints.py index 883109d572..c076b90100 100644 --- a/src/dstack/_internal/server/routers/endpoints.py +++ b/src/dstack/_internal/server/routers/endpoints.py @@ -5,7 +5,7 @@ import dstack._internal.server.services.endpoints as endpoints_services from dstack._internal.core.errors import ResourceNotExistsError -from dstack._internal.core.models.endpoint_presets import EndpointPreset, EndpointPresetDetails +from dstack._internal.core.models.endpoint_presets import EndpointPreset from dstack._internal.core.models.endpoints import Endpoint, EndpointPlan from dstack._internal.server.db import get_session from dstack._internal.server.models import ProjectModel, UserModel @@ -156,15 +156,13 @@ async def list_endpoint_presets( return CustomORJSONResponse([endpoint_preset_to_api_model(preset) for preset in presets]) -@project_router.post( - "/presets/get", summary="Get endpoint preset", response_model=EndpointPresetDetails -) +@project_router.post("/presets/get", summary="Get endpoint preset", response_model=EndpointPreset) async def get_endpoint_preset( body: GetEndpointPresetRequest, user_project: Tuple[UserModel, ProjectModel] = Depends(ProjectMember()), ): _, project = user_project - preset = await get_endpoint_preset_service().get_preset(project.name, body.name) + preset = await get_endpoint_preset_service().get_preset(project.name, body.model) if preset is None: raise ResourceNotExistsError() return CustomORJSONResponse(endpoint_preset_to_api_details(preset)) @@ -178,7 +176,7 @@ async def delete_endpoint_presets( _, project = user_project preset_service = get_endpoint_preset_service() try: - for name in body.names: - await preset_service.delete_preset(project.name, name) + for model in body.models: + await preset_service.delete_preset(project.name, model) except FileNotFoundError: raise ResourceNotExistsError() diff --git a/src/dstack/_internal/server/schemas/endpoint_presets.py b/src/dstack/_internal/server/schemas/endpoint_presets.py index 9e46813baa..91de176094 100644 --- a/src/dstack/_internal/server/schemas/endpoint_presets.py +++ b/src/dstack/_internal/server/schemas/endpoint_presets.py @@ -4,8 +4,8 @@ class GetEndpointPresetRequest(CoreModel): - name: str + model: str class DeleteEndpointPresetsRequest(CoreModel): - names: List[str] + models: List[str] diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py index 2f8a9c3faf..b6ea63cf54 100644 --- a/src/dstack/_internal/server/services/endpoints/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -2,24 +2,24 @@ from datetime import datetime from typing import Any, List, Optional -from sqlalchemy import and_, func, or_, select +from sqlalchemy import and_, exists, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError -from dstack._internal.core.models.common import ApplyAction +from dstack._internal.core.models.common import ApplyAction, EntityReference from dstack._internal.core.models.endpoints import ( Endpoint, EndpointConfiguration, EndpointPlan, EndpointPlanJobOffers, - EndpointPlanReplicaSpecGroup, EndpointPresetPolicy, EndpointProvisioningPlanAgent, EndpointProvisioningPlanNone, EndpointProvisioningPlanPreset, EndpointStatus, ) +from dstack._internal.core.models.fleets import FleetStatus from dstack._internal.core.models.runs import ServiceSpec from dstack._internal.core.models.users import GlobalRole, ProjectRole from dstack._internal.core.services import validate_dstack_resource_name @@ -27,6 +27,9 @@ from dstack._internal.server.models import ( EndpointModel, EndpointRunSubmissionModel, + ExportedFleetModel, + FleetModel, + ImportModel, ProjectModel, UserModel, ) @@ -35,7 +38,6 @@ AgentPlan, get_agent_service, get_agent_unavailable_reason, - get_effective_max_agent_budget, ) from dstack._internal.server.services.endpoints.planning import ( EndpointPresetPlan, @@ -55,6 +57,13 @@ _ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE = ( "Creating endpoint presets with the server agent requires project admin permissions." ) +_ENDPOINT_NO_FLEETS_MESSAGE = ( + "The project has no fleets. Create one before submitting an endpoint." +) +_ENDPOINT_NO_MATCHING_FLEETS_MESSAGE = ( + "No fleets match the endpoint configuration. Create a fleet or update `fleets` before " + "submitting an endpoint." +) def switch_endpoint_status( @@ -273,6 +282,24 @@ async def get_endpoint_plan( ) if current_resource is not None and current_resource.status.is_finished(): current_resource = None + has_usable_fleets = await has_endpoint_existing_usable_fleets( + session=session, + project=project, + configuration=configuration, + ) + if not has_usable_fleets: + return EndpointPlan( + project_name=project.name, + user=user.name, + configuration=configuration, + configuration_path=configuration_path, + current_resource=current_resource, + action=ApplyAction.CREATE if current_resource is None else ApplyAction.UPDATE, + preset_policy=configuration.preset_policy, + provisioning_plan=EndpointProvisioningPlanNone( + reason=get_endpoint_no_fleets_message(configuration) + ), + ) preset_plan = None unprovisionable_preset_plan = None if configuration.preset_policy != EndpointPresetPolicy.CREATE: @@ -300,7 +327,6 @@ async def get_endpoint_plan( if can_use_endpoint_agent(user=user, project=project): provisioning_plan = _agent_plan_to_provisioning_plan( get_agent_service().get_plan(), - max_budget=get_effective_max_agent_budget(configuration), reason=_get_unprovisionable_preset_reason(unprovisionable_preset_plan), ) else: @@ -330,14 +356,71 @@ def get_endpoint_agent_admin_required_message() -> str: return _ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE +def get_endpoint_no_fleets_message(configuration: EndpointConfiguration) -> str: + if configuration.fleets: + return _ENDPOINT_NO_MATCHING_FLEETS_MESSAGE + return _ENDPOINT_NO_FLEETS_MESSAGE + + +async def has_endpoint_existing_usable_fleets( + *, + session: AsyncSession, + project: ProjectModel, + configuration: EndpointConfiguration, +) -> bool: + filters = [ + FleetModel.deleted == False, + FleetModel.status == FleetStatus.ACTIVE, + _get_endpoint_usable_fleet_ownership_filter(project), + ] + if configuration.fleets is not None: + fleet_filter = _get_endpoint_configured_fleets_filter(configuration.fleets, project) + if fleet_filter is None: + return False + filters.append(fleet_filter) + res = await session.execute( + select(FleetModel.id).join(FleetModel.project).where(*filters).limit(1) + ) + return res.scalar_one_or_none() is not None + + +def _get_endpoint_usable_fleet_ownership_filter(project: ProjectModel): + is_fleet_imported_subquery = exists().where( + ImportModel.project_id == project.id, + ImportModel.export_id == ExportedFleetModel.export_id, + ExportedFleetModel.fleet_id == FleetModel.id, + ) + return or_(FleetModel.project_id == project.id, is_fleet_imported_subquery) + + +def _get_endpoint_configured_fleets_filter(fleets, project: ProjectModel): + conditions = [] + for ref in map(EntityReference.parse, fleets): + if ref.project is None: + conditions.append( + and_( + FleetModel.name == ref.name, + FleetModel.project_id == project.id, + ) + ) + else: + conditions.append( + and_( + FleetModel.name == ref.name, + ProjectModel.name == ref.project, + ) + ) + if not conditions: + return None + return or_(*conditions) + + def _agent_plan_to_provisioning_plan( agent_plan: AgentPlan, - max_budget: Optional[float], reason: Optional[str] = None, ) -> EndpointProvisioningPlanAgent: return EndpointProvisioningPlanAgent( agent_model=agent_plan.model, - max_budget=max_budget, reason=reason, ) @@ -395,7 +478,10 @@ def _get_unprovisionable_preset_reason( ) -> Optional[str]: if preset_plan is None: return None - return f"Endpoint preset {preset_plan.preset.name} matched but has no available offers." + return ( + f"Endpoint preset for model {preset_plan.preset.model} " + f"recipe {preset_plan.recipe.id} matched but has no available offers." + ) def _endpoint_preset_plan_to_provisioning_plan( @@ -404,16 +490,9 @@ def _endpoint_preset_plan_to_provisioning_plan( run_spec = preset_plan.run_plan.get_effective_run_spec() service_name = run_spec.run_name or run_spec.configuration.name or "(generated)" return EndpointProvisioningPlanPreset( - preset_name=preset_plan.preset.name, + preset_model=preset_plan.preset.model, + recipe_id=preset_plan.recipe.id, service_name=service_name, - replica_spec_groups=[ - EndpointPlanReplicaSpecGroup( - name=group.name, - resources=group.resources, - tested_resources=group.tested_resources, - ) - for group in preset_plan.preset.replica_spec_groups - ], job_offers=[ EndpointPlanJobOffers( replica_group=job_plan.job_spec.replica_group, @@ -437,12 +516,6 @@ async def create_endpoint( pipeline_hinter: PipelineHinterProtocol, ) -> Endpoint: _validate_endpoint_configuration(configuration) - await _check_can_submit_endpoint_configuration( - session=session, - project=project, - user=user, - configuration=configuration, - ) lock_namespace = f"endpoint_names_{project.name}" if is_db_sqlite(): @@ -463,6 +536,12 @@ async def create_endpoint( if endpoint_model is not None: if not endpoint_model.status.is_finished(): raise ResourceExistsError() + await _check_can_submit_endpoint_configuration( + session=session, + project=project, + user=user, + configuration=configuration, + ) endpoint_model.user = user endpoint_model.service_run_id = None endpoint_model.service_run = None @@ -487,6 +566,13 @@ async def create_endpoint( else: configuration.name = await generate_endpoint_name(session=session, project=project) + await _check_can_submit_endpoint_configuration( + session=session, + project=project, + user=user, + configuration=configuration, + ) + endpoint_model = EndpointModel( id=uuid.uuid4(), name=configuration.name, @@ -515,10 +601,15 @@ async def _check_can_submit_endpoint_configuration( user: UserModel, configuration: EndpointConfiguration, ) -> None: + if not await has_endpoint_existing_usable_fleets( + session=session, + project=project, + configuration=configuration, + ): + raise ServerClientError(get_endpoint_no_fleets_message(configuration)) if configuration.preset_policy == EndpointPresetPolicy.REUSE: return - if can_use_endpoint_agent(user=user, project=project): - return + preset_planning_result = None if configuration.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE: preset_planning_result = await find_preset_planning_result( session=session, @@ -529,6 +620,8 @@ async def _check_can_submit_endpoint_configuration( ) if preset_planning_result.provisionable is not None: return + if can_use_endpoint_agent(user=user, project=project): + return raise ForbiddenError(_ENDPOINT_AGENT_ADMIN_REQUIRED_MESSAGE) diff --git a/src/dstack/_internal/server/services/endpoints/agent/__init__.py b/src/dstack/_internal/server/services/endpoints/agent/__init__.py index 7d70a941ba..ef05a324bc 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/agent/__init__.py @@ -3,7 +3,6 @@ from dataclasses import dataclass from typing import Optional -from dstack._internal.core.models.endpoints import EndpointConfiguration from dstack._internal.server import settings from dstack._internal.server.models import EndpointModel from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport @@ -19,7 +18,8 @@ class AgentPlan: class AgentProvisioningResult: run_id: Optional[uuid.UUID] = None run_name: Optional[str] = None - candidate_run_ids: tuple[uuid.UUID, ...] = () + submitted_run_ids: tuple[uuid.UUID, ...] = () + submitted_run_names: tuple[str, ...] = () error: Optional[str] = None final_report: Optional[AgentFinalReport] = None in_progress: bool = False @@ -42,6 +42,9 @@ async def provision_endpoint( ) -> AgentProvisioningResult: pass + async def abort_endpoint(self, endpoint_model: EndpointModel) -> bool: + return True + class DisabledAgentService(AgentService): def __init__(self, reason: Optional[str] = None) -> None: @@ -61,6 +64,14 @@ async def provision_endpoint( return AgentProvisioningResult(error=self._reason or get_agent_unavailable_reason()) +async def abort_agent_endpoint(endpoint_model: EndpointModel) -> bool: + # Cancellation must work even if a new agent session cannot be started because + # the API key or Claude executable is no longer configured. + from dstack._internal.server.services.endpoints.agent.claude import ClaudeAgentService + + return await ClaudeAgentService().abort_endpoint(endpoint_model) + + def get_agent_service() -> AgentService: if settings.AGENT_ANTHROPIC_API_KEY: from dstack._internal.server.services.endpoints.agent.claude import ( @@ -83,11 +94,3 @@ def get_agent_unavailable_reason() -> Optional[str]: ) return get_claude_agent_unavailable_reason() - - -def get_effective_max_agent_budget( - configuration: EndpointConfiguration, -) -> Optional[float]: - if configuration.max_agent_budget is not None: - return configuration.max_agent_budget - return settings.AGENT_ANTHROPIC_MAX_BUDGET diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index 56523483d8..d114d53921 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -1,10 +1,13 @@ import asyncio import enum +import hashlib import json import os import shutil +import signal import socket import subprocess +import sys from contextlib import suppress from dataclasses import dataclass from datetime import datetime, timezone @@ -14,16 +17,20 @@ import yaml from pydantic import ValidationError -from sqlalchemy import func, select +from sqlalchemy import exists, func, or_, select from dstack._internal.core.models.config import GlobalConfig, ProjectConfig from dstack._internal.core.models.endpoints import EndpointConfiguration +from dstack._internal.core.models.fleets import FleetStatus from dstack._internal.server import settings from dstack._internal.server.db import get_session_ctx from dstack._internal.server.models import ( - EndpointAgentAttemptModel, - EndpointAgentAttemptStatus, + EndpointAgentSessionModel, + EndpointAgentSessionStatus, EndpointModel, + ExportedFleetModel, + FleetModel, + ImportModel, ProjectModel, ) from dstack._internal.server.schemas.runner import LogEvent @@ -32,7 +39,6 @@ AgentPlan, AgentProvisioningResult, AgentService, - get_effective_max_agent_budget, ) from dstack._internal.server.services.endpoints.agent.report import ( AGENT_FINAL_REPORT_JSON_SCHEMA, @@ -67,11 +73,28 @@ _MAX_AGENT_LOG_MESSAGE_CHARS = 4_000 _AGENT_LOG_BATCH_SIZE = 1 _AGENT_PROGRESS_LOG_NAME = "progress.jsonl" +_AGENT_SUBMISSIONS_LOG_NAME = "submissions.jsonl" _AGENT_PROCESS_STATE_NAME = "agent_process.json" _AGENT_STDOUT_LOG_NAME = "agent_stdout.jsonl" _AGENT_STDERR_LOG_NAME = "agent_stderr.jsonl" _AGENT_PROGRESS_POLL_SECONDS = 1.0 +_AGENT_PROCESS_ABORT_GRACE_SECONDS = 1.0 _CLAUDE_AGENT_TOOLS = "Bash,Read,Write,Edit,WebFetch,WebSearch,StructuredOutput" +_AGENT_ABORT_MESSAGE = "Endpoint stop requested" +_AGENT_BIN_DIR_NAME = "bin" +_AGENT_PROGRESS_ENV = "DSTACK_ENDPOINT_PROGRESS_LOG" +_ENDPOINT_NO_FLEETS_MESSAGE = ( + "The project has no fleets. Create one before submitting an endpoint." +) +_ENDPOINT_NO_MATCHING_FLEETS_MESSAGE = ( + "No fleets match the endpoint configuration. Create a fleet or update `fleets` before " + "submitting an endpoint." +) +_AGENT_START_PROGRESS_TEMPLATE = ( + "Starting endpoint prototyping agent for {model}. Allowed fleets: {fleets}. " + "The agent will inspect offers, choose a service recipe, deploy it, and verify " + "the model API before the endpoint becomes active." +) @dataclass(frozen=True) @@ -84,10 +107,22 @@ class _AgentRunnerResult: class _AgentProcessOutput: report_data: Optional[dict[str, Any]] = None result_error: Optional[str] = None - spent_budget: Optional[float] = None stdout_tail: str = "" +@dataclass(frozen=True) +class _AgentProcessState: + pid: int + pgid: int + host: Optional[str] + + +@dataclass(frozen=True) +class _EndpointAgentConstraints: + prompt_text: str + allowed_fleets: tuple[str, ...] + + class ClaudeAgentService(AgentService): def __init__( self, @@ -115,17 +150,25 @@ async def provision_endpoint( return AgentProvisioningResult(error=unavailable_reason) try: - max_agent_budget = get_effective_max_agent_budget( - EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) + configuration = EndpointConfiguration.__response__.parse_raw( + endpoint_model.configuration + ) + agent_constraints = await _get_endpoint_agent_constraints( + endpoint_model=endpoint_model, + configuration=configuration, ) - attempt = await _get_or_create_agent_attempt( + if not agent_constraints.allowed_fleets: + return AgentProvisioningResult( + error=_get_endpoint_no_fleets_message(configuration) + ) + agent_session = await _get_or_create_agent_session( endpoint_model=endpoint_model, workspace_base_dir=self._workspace_base_dir, - max_agent_budget=max_agent_budget, ) workspace = _prepare_workspace( endpoint_model=endpoint_model, - workspace_root_dir=Path(attempt.workspace_path), + workspace_root_dir=Path(agent_session.workspace_path), + agent_constraints=agent_constraints, ) except Exception as e: logger.warning("Failed to prepare endpoint agent workspace: %s", e, exc_info=True) @@ -135,43 +178,80 @@ async def provision_endpoint( if self._runner is not None: return await _run_injected_agent_runner( - attempt=attempt, + agent_session=agent_session, workspace=workspace, runner=self._runner, ) - return await _reconcile_detached_agent_attempt( - attempt=attempt, + return await _reconcile_detached_agent_session( + agent_session=agent_session, + workspace=workspace, + ) + + async def abort_endpoint(self, endpoint_model: EndpointModel) -> bool: + agent_session = await _get_latest_agent_session(endpoint_model) + if agent_session is None or agent_session.status != EndpointAgentSessionStatus.RUNNING: + return True + try: + workspace = _prepare_workspace( + endpoint_model=endpoint_model, + workspace_root_dir=Path(agent_session.workspace_path), + install_skills=False, + ) + except Exception: + logger.warning( + "Failed to prepare endpoint agent workspace for abort: endpoint=%s", + endpoint_model.name, + exc_info=True, + ) + return False + + aborted = await _abort_agent_session_process( + agent_session=agent_session, workspace=workspace, ) + if not aborted: + return False + workspace.artifacts.record_error(_AGENT_ABORT_MESSAGE) + await _mark_agent_session_failed(agent_session, _AGENT_ABORT_MESSAGE) + await _flush_agent_logs(workspace) + return True async def _run_injected_agent_runner( *, - attempt: EndpointAgentAttemptModel, + agent_session: EndpointAgentSessionModel, workspace: "_AgentWorkspace", runner: Callable[["_AgentWorkspace", dict[str, Any]], Awaitable[_AgentRunnerResult]], ) -> AgentProvisioningResult: - reconcile_result = await _reconcile_agent_artifacts(attempt=attempt, workspace=workspace) + reconcile_result = await _reconcile_agent_artifacts( + agent_session=agent_session, + workspace=workspace, + ) if reconcile_result is not None: return reconcile_result runner_result = await runner(workspace, _build_agent_request(workspace)) - candidate_run_ids = _load_candidate_run_ids(workspace) + submissions = _load_submissions(workspace) if runner_result.error is not None: workspace.artifacts.record_error(runner_result.error) - await _mark_agent_attempt_failed(attempt, runner_result.error) + await _mark_agent_session_failed(agent_session, runner_result.error) return AgentProvisioningResult( error=runner_result.error, - candidate_run_ids=candidate_run_ids, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, ) report = runner_result.report if report is None: - error = "Server agent did not return a verification report" + error = "Server agent did not return a final report" workspace.artifacts.record_error(error) - await _mark_agent_attempt_failed(attempt, error) - return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) + await _mark_agent_session_failed(agent_session, error) + return AgentProvisioningResult( + error=error, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, + ) workspace.artifacts.record_report(report) - await _mark_agent_attempt_from_report(attempt, report, spent_budget=None) + await _mark_agent_session_from_report(agent_session, report) logger.info( "Endpoint agent finished for endpoint %s: success=%s run_id=%s run_name=%s", @@ -183,158 +263,197 @@ async def _run_injected_agent_runner( return AgentProvisioningResult( run_id=report.run_id, run_name=report.run_name, - candidate_run_ids=candidate_run_ids, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, final_report=report, ) -async def _reconcile_detached_agent_attempt( +async def _reconcile_detached_agent_session( *, - attempt: EndpointAgentAttemptModel, + agent_session: EndpointAgentSessionModel, workspace: "_AgentWorkspace", ) -> AgentProvisioningResult: - reconcile_result = await _reconcile_agent_artifacts(attempt=attempt, workspace=workspace) + reconcile_result = await _reconcile_agent_artifacts( + agent_session=agent_session, + workspace=workspace, + ) if reconcile_result is not None: return reconcile_result - running_pid = _get_running_agent_process_pid(workspace, attempt=attempt) + running_pid = _get_running_agent_process_pid(workspace, agent_session=agent_session) if running_pid is not None: logger.info( "Endpoint agent process %s is already running for endpoint %s", running_pid, workspace.endpoint_name, ) + submissions = _load_submissions(workspace) return AgentProvisioningResult( in_progress=True, - candidate_run_ids=_load_candidate_run_ids(workspace), + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, ) - if attempt.pid is not None: - error = "Server agent process exited without a verification report" - workspace.artifacts.record_error(error) - await _mark_agent_attempt_failed(attempt, error) - return AgentProvisioningResult( - error=error, - candidate_run_ids=_load_candidate_run_ids(workspace), + if agent_session.pid is not None: + logger.info( + "Endpoint agent process %s is no longer running for endpoint %s; " + "resuming session %s from workspace", + agent_session.pid, + workspace.endpoint_name, + agent_session.session_num, ) + _write_trace_record( + workspace, + { + "type": "agent-process-resume", + "pid": agent_session.pid, + "process_host": agent_session.process_host, + "session_num": agent_session.session_num, + }, + ) + await _write_agent_session_progress( + agent_session, + workspace, + "Resuming endpoint prototyping agent from the existing workspace.", + ) + await _clear_agent_session_process(agent_session) try: + await _write_agent_session_progress( + agent_session, + workspace, + _get_agent_start_progress_message(workspace), + ) pid = _start_agent_subprocess_detached(workspace, _build_agent_request(workspace)) except Exception as e: logger.warning("Failed to start endpoint agent process: %s", e, exc_info=True) error = f"Failed to start endpoint agent process: {e}" workspace.artifacts.record_error(error) - await _mark_agent_attempt_failed(attempt, error) + await _mark_agent_session_failed(agent_session, error) return AgentProvisioningResult(error=error) - await _mark_agent_attempt_running(attempt, pid=pid, process_host=_get_process_host()) + await _mark_agent_session_running(agent_session, pid=pid, process_host=_get_process_host()) return AgentProvisioningResult(in_progress=True) async def _reconcile_agent_artifacts( *, - attempt: EndpointAgentAttemptModel, + agent_session: EndpointAgentSessionModel, workspace: "_AgentWorkspace", ) -> Optional[AgentProvisioningResult]: - process_output = await _reconcile_agent_stream_files(attempt=attempt, workspace=workspace) - await _reconcile_agent_progress(attempt=attempt, workspace=workspace) - candidate_run_ids = _load_candidate_run_ids(workspace) + process_output = await _reconcile_agent_stream_files( + agent_session=agent_session, + workspace=workspace, + ) + await _reconcile_agent_progress(agent_session=agent_session, workspace=workspace) + submissions = _load_submissions(workspace) report = _load_final_report(workspace) if report is None and process_output.report_data is not None: try: report = AgentFinalReport.parse_obj(process_output.report_data) except ValidationError as e: - error = f"Server agent returned an invalid verification report: {e}" + error = f"Server agent returned an invalid final report: {e}" workspace.artifacts.record_error(error) - await _mark_agent_attempt_failed(attempt, error) - return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) + await _mark_agent_session_failed(agent_session, error) + return AgentProvisioningResult( + error=error, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, + ) if report is not None: - if ( - process_output.spent_budget is None - and _get_running_agent_process_pid(workspace, attempt=attempt) is not None - ): + if _get_running_agent_process_pid(workspace, agent_session=agent_session) is not None: return AgentProvisioningResult( in_progress=True, - candidate_run_ids=candidate_run_ids, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, ) workspace.artifacts.record_report(report) - await _mark_agent_attempt_from_report( - attempt, - report, - spent_budget=process_output.spent_budget, - ) + await _mark_agent_session_from_report(agent_session, report) return AgentProvisioningResult( run_id=report.run_id, run_name=report.run_name, - candidate_run_ids=candidate_run_ids, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, final_report=report, ) error = _load_agent_error(workspace) if error is not None: - await _mark_agent_attempt_failed( - attempt, - error, - spent_budget=process_output.spent_budget, + await _mark_agent_session_failed(agent_session, error) + return AgentProvisioningResult( + error=error, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, ) - return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) if process_output.result_error is not None: - error = "Server agent failed before returning a verification report" + error = "Server agent failed before returning a final report" workspace.artifacts.record_error(error) - await _mark_agent_attempt_failed( - attempt, - error, - spent_budget=process_output.spent_budget, + await _mark_agent_session_failed(agent_session, error) + return AgentProvisioningResult( + error=error, + submitted_run_ids=submissions.run_ids, + submitted_run_names=submissions.run_names, ) - return AgentProvisioningResult(error=error, candidate_run_ids=candidate_run_ids) return None -async def _get_or_create_agent_attempt( +async def _get_or_create_agent_session( *, endpoint_model: EndpointModel, workspace_base_dir: Path, - max_agent_budget: Optional[float], -) -> EndpointAgentAttemptModel: - async with get_session_ctx() as session: - res = await session.execute( - select(EndpointAgentAttemptModel) - .where(EndpointAgentAttemptModel.endpoint_id == endpoint_model.id) - .order_by(EndpointAgentAttemptModel.attempt_num.desc()) +) -> EndpointAgentSessionModel: + async with get_session_ctx() as db_session: + res = await db_session.execute( + select(EndpointAgentSessionModel) + .where(EndpointAgentSessionModel.endpoint_id == endpoint_model.id) + .order_by(EndpointAgentSessionModel.session_num.desc()) .limit(1) ) - attempt = res.scalar_one_or_none() - if attempt is not None and attempt.created_at >= endpoint_model.created_at: - return attempt + agent_session = res.scalar_one_or_none() + if agent_session is not None and agent_session.created_at >= endpoint_model.created_at: + return agent_session - max_attempt_num_res = await session.execute( - select(func.max(EndpointAgentAttemptModel.attempt_num)).where( - EndpointAgentAttemptModel.endpoint_id == endpoint_model.id + max_session_num_res = await db_session.execute( + select(func.max(EndpointAgentSessionModel.session_num)).where( + EndpointAgentSessionModel.endpoint_id == endpoint_model.id ) ) - attempt_num = (max_attempt_num_res.scalar_one_or_none() or 0) + 1 + session_num = (max_session_num_res.scalar_one_or_none() or 0) + 1 now = get_current_datetime() legacy_workspace_path = workspace_base_dir / str(endpoint_model.id) - if attempt_num == 1 and _has_agent_workspace_artifacts(legacy_workspace_path): + if session_num == 1 and _has_agent_workspace_artifacts(legacy_workspace_path): workspace_path = legacy_workspace_path else: - workspace_path = workspace_base_dir / str(endpoint_model.id) / str(attempt_num) - attempt = EndpointAgentAttemptModel( + workspace_path = workspace_base_dir / str(endpoint_model.id) / str(session_num) + agent_session = EndpointAgentSessionModel( endpoint_id=endpoint_model.id, - attempt_num=attempt_num, - status=EndpointAgentAttemptStatus.RUNNING, + session_num=session_num, + status=EndpointAgentSessionStatus.RUNNING, workspace_path=str(workspace_path), progress_log_offset=0, stdout_log_offset=0, stderr_log_offset=0, - max_agent_budget=max_agent_budget, created_at=now, updated_at=now, ) - session.add(attempt) - await session.commit() - return attempt + db_session.add(agent_session) + await db_session.commit() + return agent_session + + +async def _get_latest_agent_session( + endpoint_model: EndpointModel, +) -> Optional[EndpointAgentSessionModel]: + async with get_session_ctx() as db_session: + res = await db_session.execute( + select(EndpointAgentSessionModel) + .where(EndpointAgentSessionModel.endpoint_id == endpoint_model.id) + .order_by(EndpointAgentSessionModel.session_num.desc()) + .limit(1) + ) + return res.scalar_one_or_none() def _has_agent_workspace_artifacts(root_dir: Path) -> bool: @@ -350,178 +469,203 @@ def _has_agent_workspace_artifacts(root_dir: Path) -> bool: ) -async def _get_agent_attempt(session, attempt: EndpointAgentAttemptModel): - return await session.get( - EndpointAgentAttemptModel, +async def _get_agent_session(db_session, agent_session: EndpointAgentSessionModel): + return await db_session.get( + EndpointAgentSessionModel, { - "endpoint_id": attempt.endpoint_id, - "attempt_num": attempt.attempt_num, + "endpoint_id": agent_session.endpoint_id, + "session_num": agent_session.session_num, }, ) -async def _mark_agent_attempt_running( - attempt: EndpointAgentAttemptModel, +async def _mark_agent_session_running( + agent_session: EndpointAgentSessionModel, *, pid: int, process_host: str, ) -> None: - async with get_session_ctx() as session: - stored_attempt = await _get_agent_attempt(session, attempt) - if stored_attempt is None: + async with get_session_ctx() as db_session: + stored_session = await _get_agent_session(db_session, agent_session) + if stored_session is None: + return + stored_session.status = EndpointAgentSessionStatus.RUNNING + stored_session.pid = pid + stored_session.process_host = process_host + stored_session.updated_at = get_current_datetime() + await db_session.commit() + agent_session.pid = pid + agent_session.process_host = process_host + + +async def _clear_agent_session_process(agent_session: EndpointAgentSessionModel) -> None: + async with get_session_ctx() as db_session: + stored_session = await _get_agent_session(db_session, agent_session) + if stored_session is None: return - stored_attempt.status = EndpointAgentAttemptStatus.RUNNING - stored_attempt.pid = pid - stored_attempt.process_host = process_host - stored_attempt.updated_at = get_current_datetime() - await session.commit() - attempt.pid = pid - attempt.process_host = process_host + stored_session.pid = None + stored_session.process_host = None + stored_session.updated_at = get_current_datetime() + await db_session.commit() + agent_session.pid = None + agent_session.process_host = None -async def _mark_agent_attempt_from_report( - attempt: EndpointAgentAttemptModel, +async def _mark_agent_session_from_report( + agent_session: EndpointAgentSessionModel, report: AgentFinalReport, - *, - spent_budget: Optional[float], ) -> None: if report.success: - await _mark_agent_attempt_succeeded(attempt, spent_budget=spent_budget) + await _mark_agent_session_succeeded(agent_session) else: - await _mark_agent_attempt_failed( - attempt, + await _mark_agent_session_failed( + agent_session, report.failure_summary or "Server agent did not verify the endpoint", - spent_budget=spent_budget, ) -async def _mark_agent_attempt_succeeded( - attempt: EndpointAgentAttemptModel, - *, - spent_budget: Optional[float], +async def _mark_agent_session_succeeded( + agent_session: EndpointAgentSessionModel, ) -> None: now = get_current_datetime() - async with get_session_ctx() as session: - stored_attempt = await _get_agent_attempt(session, attempt) - if stored_attempt is None: + async with get_session_ctx() as db_session: + stored_session = await _get_agent_session(db_session, agent_session) + if stored_session is None: return - stored_attempt.status = EndpointAgentAttemptStatus.SUCCEEDED - stored_attempt.status_message = None - if spent_budget is not None: - stored_attempt.spent_agent_budget = spent_budget - stored_attempt.finished_at = stored_attempt.finished_at or now - stored_attempt.updated_at = now - await session.commit() - attempt.status = EndpointAgentAttemptStatus.SUCCEEDED - attempt.status_message = None - attempt.spent_agent_budget = stored_attempt.spent_agent_budget - attempt.finished_at = stored_attempt.finished_at - - -async def _mark_agent_attempt_failed( - attempt: EndpointAgentAttemptModel, + stored_session.status = EndpointAgentSessionStatus.SUCCEEDED + stored_session.status_message = None + stored_session.finished_at = stored_session.finished_at or now + stored_session.updated_at = now + await db_session.commit() + agent_session.status = EndpointAgentSessionStatus.SUCCEEDED + agent_session.status_message = None + agent_session.finished_at = stored_session.finished_at + + +async def _mark_agent_session_failed( + agent_session: EndpointAgentSessionModel, error: str, - *, - spent_budget: Optional[float] = None, ) -> None: now = get_current_datetime() - async with get_session_ctx() as session: - stored_attempt = await _get_agent_attempt(session, attempt) - if stored_attempt is None: + async with get_session_ctx() as db_session: + stored_session = await _get_agent_session(db_session, agent_session) + if stored_session is None: return - stored_attempt.status = EndpointAgentAttemptStatus.FAILED - stored_attempt.status_message = error - if spent_budget is not None: - stored_attempt.spent_agent_budget = spent_budget - stored_attempt.finished_at = stored_attempt.finished_at or now - stored_attempt.updated_at = now - await session.commit() - attempt.status = EndpointAgentAttemptStatus.FAILED - attempt.status_message = error - attempt.spent_agent_budget = stored_attempt.spent_agent_budget - attempt.finished_at = stored_attempt.finished_at - - -async def _update_agent_attempt_offsets( - attempt: EndpointAgentAttemptModel, + stored_session.status = EndpointAgentSessionStatus.FAILED + stored_session.status_message = error + stored_session.finished_at = stored_session.finished_at or now + stored_session.updated_at = now + await db_session.commit() + agent_session.status = EndpointAgentSessionStatus.FAILED + agent_session.status_message = error + agent_session.finished_at = stored_session.finished_at + + +async def _update_agent_session_offsets( + agent_session: EndpointAgentSessionModel, *, progress_log_offset: Optional[int] = None, stdout_log_offset: Optional[int] = None, stderr_log_offset: Optional[int] = None, - spent_budget: Optional[float] = None, ) -> None: - async with get_session_ctx() as session: - stored_attempt = await _get_agent_attempt(session, attempt) - if stored_attempt is None: + async with get_session_ctx() as db_session: + stored_session = await _get_agent_session(db_session, agent_session) + if stored_session is None: return if progress_log_offset is not None: - stored_attempt.progress_log_offset = progress_log_offset - attempt.progress_log_offset = progress_log_offset + stored_session.progress_log_offset = progress_log_offset + agent_session.progress_log_offset = progress_log_offset if stdout_log_offset is not None: - stored_attempt.stdout_log_offset = stdout_log_offset - attempt.stdout_log_offset = stdout_log_offset + stored_session.stdout_log_offset = stdout_log_offset + agent_session.stdout_log_offset = stdout_log_offset if stderr_log_offset is not None: - stored_attempt.stderr_log_offset = stderr_log_offset - attempt.stderr_log_offset = stderr_log_offset - if spent_budget is not None: - stored_attempt.spent_agent_budget = spent_budget - attempt.spent_agent_budget = spent_budget - stored_attempt.updated_at = get_current_datetime() - await session.commit() + stored_session.stderr_log_offset = stderr_log_offset + agent_session.stderr_log_offset = stderr_log_offset + stored_session.updated_at = get_current_datetime() + await db_session.commit() async def _reconcile_agent_progress( *, - attempt: EndpointAgentAttemptModel, + agent_session: EndpointAgentSessionModel, workspace: "_AgentWorkspace", ) -> None: offset = await _flush_agent_progress_logs( workspace, - offset=attempt.progress_log_offset, + offset=agent_session.progress_log_offset, include_partial=False, ) - if offset != attempt.progress_log_offset: - await _update_agent_attempt_offsets(attempt, progress_log_offset=offset) + if offset != agent_session.progress_log_offset: + await _update_agent_session_offsets(agent_session, progress_log_offset=offset) + + +async def _write_agent_session_progress( + agent_session: EndpointAgentSessionModel, + workspace: "_AgentWorkspace", + message: str, +) -> None: + _append_agent_progress_record(workspace, message) + offset = workspace.progress_path.stat().st_size + await _write_agent_log(workspace, message) + await _flush_agent_logs(workspace) + await _update_agent_session_offsets(agent_session, progress_log_offset=offset) + + +def _append_agent_progress_record(workspace: "_AgentWorkspace", message: str) -> None: + workspace.progress_path.parent.mkdir(parents=True, exist_ok=True) + with workspace.progress_path.open("a", encoding="utf-8") as f: + f.write(json.dumps({"message": message}, ensure_ascii=True) + "\n") + + +def _get_agent_start_progress_message(workspace: "_AgentWorkspace") -> str: + fleets = ", ".join(workspace.allowed_fleets) if workspace.allowed_fleets else "none" + return _AGENT_START_PROGRESS_TEMPLATE.format(model=workspace.model, fleets=fleets) async def _reconcile_agent_stream_files( *, - attempt: EndpointAgentAttemptModel, + agent_session: EndpointAgentSessionModel, workspace: "_AgentWorkspace", ) -> _AgentProcessOutput: stdout_output, stdout_offset = await _read_agent_stream_file( workspace=workspace, path=workspace.work_dir / _AGENT_STDOUT_LOG_NAME, - offset=attempt.stdout_log_offset, + offset=agent_session.stdout_log_offset, stream_name="stdout", ) stderr_output, stderr_offset = await _read_agent_stream_file( workspace=workspace, path=workspace.work_dir / _AGENT_STDERR_LOG_NAME, - offset=attempt.stderr_log_offset, + offset=agent_session.stderr_log_offset, stream_name="stderr", ) output = _merge_process_outputs(stdout_output, stderr_output) if ( - stdout_offset != attempt.stdout_log_offset - or stderr_offset != attempt.stderr_log_offset - or output.spent_budget is not None + stdout_offset != agent_session.stdout_log_offset + or stderr_offset != agent_session.stderr_log_offset ): - await _update_agent_attempt_offsets( - attempt, + await _update_agent_session_offsets( + agent_session, stdout_log_offset=stdout_offset, stderr_log_offset=stderr_offset, - spent_budget=output.spent_budget, ) return output -def _load_candidate_run_ids(workspace: "_AgentWorkspace") -> tuple[UUID, ...]: - path = workspace.work_dir / "candidates.jsonl" +@dataclass(frozen=True) +class _AgentSubmissions: + run_ids: tuple[UUID, ...] = () + run_names: tuple[str, ...] = () + + +def _load_submissions(workspace: "_AgentWorkspace") -> _AgentSubmissions: + path = workspace.work_dir / _AGENT_SUBMISSIONS_LOG_NAME if not path.exists(): - return () + return _AgentSubmissions() run_ids: list[UUID] = [] - seen: set[UUID] = set() + run_names: list[str] = [] + seen_run_ids: set[UUID] = set() + seen_run_names: set[str] = set() for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): if not line.strip(): continue @@ -530,11 +674,17 @@ def _load_candidate_run_ids(workspace: "_AgentWorkspace") -> tuple[UUID, ...]: except json.JSONDecodeError: _write_trace_record( workspace, - {"type": "agent-candidate-invalid", "line": line}, + {"type": "agent-submission-invalid", "path": str(path), "line": line}, ) continue if not isinstance(parsed, dict): continue + raw_name = parsed.get("name") + if isinstance(raw_name, str): + run_name = raw_name.strip() + if run_name and run_name not in seen_run_names: + seen_run_names.add(run_name) + run_names.append(run_name) raw_run_id = parsed.get("run_id") if not isinstance(raw_run_id, str) or not raw_run_id.strip(): continue @@ -543,13 +693,17 @@ def _load_candidate_run_ids(workspace: "_AgentWorkspace") -> tuple[UUID, ...]: except ValueError: _write_trace_record( workspace, - {"type": "agent-candidate-invalid-run-id", "run_id": raw_run_id}, + { + "type": "agent-submission-invalid-run-id", + "path": str(path), + "run_id": raw_run_id, + }, ) continue - if run_id not in seen: - seen.add(run_id) + if run_id not in seen_run_ids: + seen_run_ids.add(run_id) run_ids.append(run_id) - return tuple(run_ids) + return _AgentSubmissions(run_ids=tuple(run_ids), run_names=tuple(run_names)) def _load_agent_error(workspace: "_AgentWorkspace") -> Optional[str]: @@ -596,10 +750,10 @@ def __init__( redacted_values: Sequence[str], endpoint_name: str, model: str, - max_agent_budget: Optional[float], endpoint_constraints: str = "", max_price: Optional[float] = None, spot_policy: Optional[str] = None, + allowed_fleets: Sequence[str] = (), log_writer: Optional["_AgentLogWriter"] = None, ) -> None: self.root_dir = root_dir @@ -610,10 +764,10 @@ def __init__( self.redacted_values = tuple(redacted_values) self.endpoint_name = endpoint_name self.model = model - self.max_agent_budget = max_agent_budget self.endpoint_constraints = endpoint_constraints self.max_price = max_price self.spot_policy = spot_policy + self.allowed_fleets = tuple(allowed_fleets) self.log_writer = log_writer self.artifacts = _AgentArtifactRecorder(self) @@ -674,16 +828,24 @@ def _prepare_workspace( *, endpoint_model: EndpointModel, workspace_root_dir: Path, + install_skills: bool = True, + agent_constraints: Optional[_EndpointAgentConstraints] = None, ) -> _AgentWorkspace: configuration = EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) + endpoint_env = configuration.env.as_dict() + if agent_constraints is None: + agent_constraints = _get_static_endpoint_agent_constraints(configuration, endpoint_env) root_dir = workspace_root_dir home_dir = root_dir / "home" + agent_home_dir = _prepare_agent_home_dir(root_dir=root_dir, home_dir=home_dir) work_dir = root_dir / "workspace" dstack_dir = home_dir / ".dstack" work_dir.mkdir(parents=True, exist_ok=True) dstack_dir.mkdir(parents=True, exist_ok=True) token = endpoint_model.user.token.get_plaintext_or_error() + allowed_fleets = agent_constraints.allowed_fleets + bin_dir = root_dir / _AGENT_BIN_DIR_NAME _write_cli_config( dstack_dir=dstack_dir, project_name=endpoint_model.project.name, @@ -691,11 +853,13 @@ def _prepare_workspace( token=token, ) - endpoint_env = configuration.env.as_dict() env = _build_agent_env( - home_dir=home_dir, + home_dir=agent_home_dir, project_name=endpoint_model.project.name, + server_url=settings.SERVER_URL, + token=token, endpoint_env=endpoint_env, + bin_dir=bin_dir, ) redacted_values = _get_redacted_values( [ @@ -715,21 +879,47 @@ def _prepare_workspace( redacted_values=redacted_values, endpoint_name=endpoint_model.name, model=configuration.model, - max_agent_budget=get_effective_max_agent_budget(configuration), - endpoint_constraints=_format_endpoint_constraints(configuration, endpoint_env), + endpoint_constraints=agent_constraints.prompt_text, max_price=configuration.max_price, spot_policy=configuration.spot_policy.value if configuration.spot_policy else None, + allowed_fleets=allowed_fleets, log_writer=_AgentLogWriter( project=endpoint_model.project, endpoint_id=endpoint_model.id, endpoint_name=endpoint_model.name, ), ) - _install_agent_skills(workspace) + workspace.env[_AGENT_PROGRESS_ENV] = str(workspace.progress_path) + _install_agent_helper_scripts(workspace) + if install_skills: + _install_agent_skills(workspace) workspace.artifacts.initialize() return workspace +def _prepare_agent_home_dir(*, root_dir: Path, home_dir: Path) -> Path: + home_dir.mkdir(parents=True, exist_ok=True) + ssh_dir = home_dir / ".ssh" + ssh_dir.mkdir(parents=True, exist_ok=True) + ssh_dir.chmod(0o700) + short_home_dir = _get_short_agent_home_dir(root_dir) + short_home_dir.parent.mkdir(parents=True, exist_ok=True) + if short_home_dir.is_symlink() and short_home_dir.resolve() == home_dir.resolve(): + return short_home_dir + if short_home_dir.exists() or short_home_dir.is_symlink(): + if short_home_dir.is_dir() and not short_home_dir.is_symlink(): + shutil.rmtree(short_home_dir) + else: + short_home_dir.unlink() + short_home_dir.symlink_to(home_dir, target_is_directory=True) + return short_home_dir + + +def _get_short_agent_home_dir(root_dir: Path) -> Path: + root_hash = hashlib.sha1(str(root_dir).encode()).hexdigest()[:8] + return Path("/tmp") / f"dah-{root_hash}" + + class _AgentArtifactRecorder: def __init__(self, workspace: _AgentWorkspace) -> None: self._workspace = workspace @@ -740,23 +930,11 @@ def initialize(self) -> None: self._command_counter = self._get_last_command_output_num() self._update_agent_state(phase="starting") for filename in [ - "sources.jsonl", - "candidates.jsonl", + _AGENT_SUBMISSIONS_LOG_NAME, "commands.jsonl", _AGENT_PROGRESS_LOG_NAME, ]: (self._workspace.work_dir / filename).touch(exist_ok=True) - hardware_reasoning_path = self._workspace.work_dir / "hardware_reasoning.md" - if not hardware_reasoning_path.exists(): - hardware_reasoning_path.write_text( - ( - "# Hardware Reasoning\n\n" - "The endpoint agent should update this file with the model size,\n" - "serving framework, resource estimate, offer/fleet choice, and\n" - "why the selected hardware is credible.\n" - ), - encoding="utf-8", - ) def mark_running(self) -> None: self._update_agent_state(phase="running") @@ -805,7 +983,6 @@ def _update_agent_state(self, phase: str) -> None: "endpoint_name": self._workspace.endpoint_name, "model": self._workspace.model, "phase": phase, - "max_agent_budget": self._workspace.max_agent_budget, "max_hourly_price": self._workspace.max_price, "spot_policy": self._workspace.spot_policy, "updated_at": now, @@ -925,20 +1102,153 @@ def _build_agent_env( *, home_dir: Path, project_name: str, + server_url: str, + token: str, endpoint_env: dict[str, str], + bin_dir: Path, ) -> dict[str, str]: env = {name: value for name in _INHERITED_ENV_NAMES if (value := os.environ.get(name))} + path = env.get("PATH", "") + env["PATH"] = str(bin_dir) if not path else os.pathsep.join([str(bin_dir), path]) env.update( { "ANTHROPIC_API_KEY": settings.AGENT_ANTHROPIC_API_KEY or "", "HOME": str(home_dir), "DSTACK_PROJECT": project_name, + "DSTACK_ENDPOINT_SERVER_URL": server_url, + "DSTACK_ENDPOINT_BEARER_TOKEN": token, } ) env.update(endpoint_env) return env +async def _get_endpoint_agent_constraints( + *, + endpoint_model: EndpointModel, + configuration: EndpointConfiguration, +) -> _EndpointAgentConstraints: + endpoint_env = configuration.env.as_dict() + allowed_fleets = await _get_endpoint_allowed_fleets( + endpoint_model=endpoint_model, + configuration=configuration, + ) + return _EndpointAgentConstraints( + prompt_text=_format_endpoint_constraints( + configuration, + endpoint_env, + allowed_fleets=allowed_fleets, + ), + allowed_fleets=allowed_fleets, + ) + + +def _get_static_endpoint_agent_constraints( + configuration: EndpointConfiguration, + endpoint_env: dict[str, str], +) -> _EndpointAgentConstraints: + allowed_fleets = _get_configured_endpoint_fleet_refs(configuration) + return _EndpointAgentConstraints( + prompt_text=_format_endpoint_constraints( + configuration, + endpoint_env, + allowed_fleets=allowed_fleets, + ), + allowed_fleets=allowed_fleets, + ) + + +async def _get_endpoint_allowed_fleets( + *, + endpoint_model: EndpointModel, + configuration: EndpointConfiguration, +) -> tuple[str, ...]: + if configuration.fleets is not None and len(configuration.fleets) == 0: + return () + configured_fleets = _get_configured_endpoint_fleet_refs(configuration) + if configured_fleets: + return configured_fleets + + async with get_session_ctx() as db_session: + is_fleet_imported_subquery = exists().where( + ImportModel.project_id == endpoint_model.project_id, + ImportModel.export_id == ExportedFleetModel.export_id, + ExportedFleetModel.fleet_id == FleetModel.id, + ) + res = await db_session.execute( + select(FleetModel.name, FleetModel.project_id, ProjectModel.name) + .join(FleetModel.project) + .where( + FleetModel.deleted == False, + FleetModel.status == FleetStatus.ACTIVE, + or_( + FleetModel.project_id == endpoint_model.project_id, + is_fleet_imported_subquery, + ), + ) + .order_by(ProjectModel.name, FleetModel.name) + ) + return tuple( + name if project_id == endpoint_model.project_id else f"{project_name}/{name}" + for name, project_id, project_name in res.all() + ) + + +def _get_configured_endpoint_fleet_refs( + configuration: EndpointConfiguration, +) -> tuple[str, ...]: + if not configuration.fleets: + return () + return tuple(_format_constraint_value(fleet) for fleet in configuration.fleets) + + +def _get_endpoint_no_fleets_message(configuration: EndpointConfiguration) -> str: + if configuration.fleets: + return _ENDPOINT_NO_MATCHING_FLEETS_MESSAGE + return _ENDPOINT_NO_FLEETS_MESSAGE + + +def _install_agent_helper_scripts(workspace: _AgentWorkspace) -> None: + bin_dir = workspace.root_dir / _AGENT_BIN_DIR_NAME + bin_dir.mkdir(parents=True, exist_ok=True) + scripts = { + "progress": _get_agent_progress_script(), + } + for name, script in scripts.items(): + script_path = bin_dir / name + script_path.write_text(script, encoding="utf-8") + script_path.chmod(0o755) + + +def _get_agent_progress_script() -> str: + return f"""#!{sys.executable} +import json +import os +from pathlib import Path +import sys + +PROGRESS_ENV = "{_AGENT_PROGRESS_ENV}" + + +def main(): + message = " ".join(sys.argv[1:]).strip() + if not message and not sys.stdin.isatty(): + message = sys.stdin.read().strip() + if not message: + print("Usage: progress ", file=sys.stderr) + return 2 + path = Path(os.environ.get(PROGRESS_ENV, "progress.jsonl")) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps({{"message": message}}, ensure_ascii=False) + "\\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +""" + + def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: return { "prompt": _build_prompt(workspace), @@ -952,7 +1262,6 @@ def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: "disallowed_tools": "Task,NotebookEdit", "model": settings.AGENT_ANTHROPIC_MODEL, "max_turns": None, - "max_budget": workspace.max_agent_budget, "json_schema": AGENT_FINAL_REPORT_JSON_SCHEMA, }, } @@ -961,25 +1270,11 @@ def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: def _build_prompt(workspace: _AgentWorkspace) -> str: return f"""{_load_endpoint_prompt()} -Deploy endpoint {workspace.endpoint_name!r} for model {workspace.model!r}. +Endpoint request: +- name: {workspace.endpoint_name} +- model: {workspace.model} {workspace.endpoint_constraints} - -Bundled Claude Code skills are installed in `.claude/skills`. Load and follow: -- `/dstack` -- `/dstack-prototyping` - -Required final service: -- service YAML type: service -- service YAML must set `model: {workspace.model}` -- service run name must not equal the endpoint name `{workspace.endpoint_name}` -- for sequential service attempts, prefer `{workspace.endpoint_name}-1`, `{workspace.endpoint_name}-2`, etc. -- submit runs detached with `dstack apply -f -y -d` -- use concise, unique run names that are useful for debugging -- the successful final report must include the verified service run `run_id` - -When you have a terminal result, write `final_report.json` in the workspace with the -same fields requested by the JSON schema, then return only the structured final report. """ @@ -1002,59 +1297,36 @@ def _install_agent_skills(workspace: _AgentWorkspace) -> None: def _get_packaged_skills_dir() -> Path: for parent in Path(__file__).resolve().parents: - candidate = parent / "skills" - if (candidate / "dstack" / "SKILL.md").is_file(): - return candidate + skills_dir = parent / "skills" + if (skills_dir / "dstack" / "SKILL.md").is_file(): + return skills_dir raise FileNotFoundError("Could not find packaged dstack skills") def _format_endpoint_constraints( configuration: EndpointConfiguration, endpoint_env: dict[str, str], + *, + allowed_fleets: Sequence[str], ) -> str: lines = [ - "Binding endpoint constraints:", - "- Every `dstack offer`, `dstack apply` preview, and submitted service must honor these constraints.", - "- Do not submit a service if the plan violates these constraints.", + "Fixed endpoint constraints:", + "- Do not submit any task or service that conflicts with these values.", + "- Put applicable fixed constraints into the final service YAML.", ] - cli_flags = [] if configuration.max_price is not None: lines.append(f"- max_price: {configuration.max_price}") - cli_flags.extend(["--max-price", str(configuration.max_price)]) if configuration.spot_policy is not None: lines.append(f"- spot_policy: {configuration.spot_policy.value}") - if configuration.spot_policy.value == "spot": - cli_flags.append("--spot") - elif configuration.spot_policy.value == "on-demand": - cli_flags.append("--on-demand") - else: - cli_flags.append("--spot-auto") - for field, flag in [ - ("backends", "--backend"), - ("regions", "--region"), - ("instance_types", "--instance-type"), - ("fleets", "--fleet"), - ]: + for field in ["backends", "regions", "availability_zones", "instance_types"]: values = getattr(configuration, field) if not values: continue formatted_values = [_format_constraint_value(value) for value in values] lines.append(f"- {field}: {', '.join(formatted_values)}") - for value in formatted_values: - cli_flags.extend([flag, value]) - for field in ["availability_zones", "instances"]: - values = getattr(configuration, field) - if not values: - continue - formatted_values = [_format_constraint_value(value) for value in values] - lines.append(f"- {field}: {', '.join(formatted_values)}") - if configuration.creation_policy is not None: - lines.append(f"- creation_policy: {configuration.creation_policy.value}") - if configuration.creation_policy.value == "reuse": - cli_flags.append("--reuse") - if cli_flags: - lines.append(f"- Reuse these CLI flags where applicable: {' '.join(cli_flags)}") - else: + if allowed_fleets: + lines.append(f"- fleets: {', '.join(allowed_fleets)}") + if len(lines) == 3: lines.append("- No explicit profile constraints were set.") if endpoint_env: lines.append( @@ -1158,25 +1430,19 @@ async def _run_agent_in_subprocess( process_output.report_data = _load_final_report_artifact(workspace) if process_output.report_data is None: if process_output.result_error is not None: - return _AgentRunnerResult( - error="Server agent failed before returning a verification report" - ) + return _AgentRunnerResult(error="Server agent failed before returning a final report") if returncode not in (0, None): return _AgentRunnerResult( error=( - "Server agent process exited without a verification report " + "Server agent process exited without a final report " f"(return code {returncode})" ) ) - return _AgentRunnerResult( - error="Server agent process exited without a verification report" - ) + return _AgentRunnerResult(error="Server agent process exited without a final report") try: return _AgentRunnerResult(report=AgentFinalReport.parse_obj(process_output.report_data)) except ValidationError as e: - return _AgentRunnerResult( - error=f"Server agent returned an invalid verification report: {e}" - ) + return _AgentRunnerResult(error=f"Server agent returned an invalid final report: {e}") def _build_claude_command(request: dict[str, Any]) -> list[str]: @@ -1204,8 +1470,6 @@ def _build_claude_command(request: dict[str, Any]) -> list[str]: ] if options["max_turns"] is not None: cmd[2:2] = ["--max-turns", str(options["max_turns"])] - if options["max_budget"] is not None: - cmd[2:2] = ["--max-budget-usd", str(options["max_budget"])] return cmd @@ -1240,7 +1504,7 @@ async def _read_agent_stream( if not line_bytes: return output line = line_bytes.decode(errors="replace") - _process_agent_stream_line( + await _process_agent_stream_line( line=line, workspace=workspace, stream_name=stream_name, @@ -1258,7 +1522,7 @@ async def _read_agent_stream_file( output = _AgentProcessOutput() lines, new_offset = _read_complete_file_lines(path=path, offset=offset) for line in lines: - _process_agent_stream_line( + await _process_agent_stream_line( line=line, workspace=workspace, stream_name=stream_name, @@ -1267,7 +1531,7 @@ async def _read_agent_stream_file( return output, new_offset -def _process_agent_stream_line( +async def _process_agent_stream_line( *, line: str, workspace: _AgentWorkspace, @@ -1348,19 +1612,11 @@ async def flush(self, *, include_partial: bool = False) -> None: async def _write_progress_line(self, line: str) -> None: if not line.strip(): return - try: - parsed = json.loads(line) - except json.JSONDecodeError: - _write_trace_record( - self._workspace, - {"type": "agent-progress-invalid", "line": line}, - ) - return - message = _format_agent_progress_log_message(parsed) + message = _parse_agent_progress_log_message(line) if message is None: _write_trace_record( self._workspace, - {"type": "agent-progress-ignored", "record": parsed}, + {"type": "agent-progress-ignored", "line": line}, ) return await _write_agent_log(self._workspace, message) @@ -1392,33 +1648,34 @@ async def _flush_agent_progress_logs( async def _write_agent_progress_line(workspace: _AgentWorkspace, line: str) -> None: if not line.strip(): return - try: - parsed = json.loads(line) - except json.JSONDecodeError: - _write_trace_record( - workspace, - {"type": "agent-progress-invalid", "line": line}, - ) - return - message = _format_agent_progress_log_message(parsed) + message = _parse_agent_progress_log_message(line) if message is None: _write_trace_record( workspace, - {"type": "agent-progress-ignored", "record": parsed}, + {"type": "agent-progress-ignored", "line": line}, ) return await _write_agent_log(workspace, message) +def _parse_agent_progress_log_message(line: str) -> Optional[str]: + try: + parsed = json.loads(line) + except json.JSONDecodeError: + message = line.strip() + return message or None + return _format_agent_progress_log_message(parsed) + + def _format_agent_progress_log_message(record: Any) -> Optional[str]: + if isinstance(record, str): + message = record.strip() + return message or None if not isinstance(record, dict): return None message = record.get("message") if not isinstance(message, str) or not message.strip(): return None - phase = record.get("phase") - if isinstance(phase, str) and phase.strip(): - return f"{phase.strip()}: {message.strip()}" return message.strip() @@ -1428,9 +1685,6 @@ def _update_agent_process_output( ) -> None: if message.get("type") != "result": return - total_cost = message.get("total_cost_usd") - if isinstance(total_cost, (int, float)): - output.spent_budget = float(total_cost) if message.get("is_error"): output.result_error = message.get("result") or "Claude agent failed" structured_output = message.get("structured_output") @@ -1456,8 +1710,6 @@ def _merge_process_outputs(*outputs: _AgentProcessOutput) -> _AgentProcessOutput merged.report_data = output.report_data if output.result_error is not None: merged.result_error = output.result_error - if output.spent_budget is not None: - merged.spent_budget = output.spent_budget merged.stdout_tail = _append_bounded_output(merged.stdout_tail, output.stdout_tail) return merged @@ -1527,6 +1779,7 @@ def _write_agent_process_state(workspace: _AgentWorkspace, pid: int) -> None: path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME data = { "pid": pid, + "pgid": pid, "host": _get_process_host(), "started_at": _utcnow_iso(), } @@ -1548,34 +1801,119 @@ def _clear_agent_process_state(workspace: _AgentWorkspace, pid: int) -> None: def _get_running_agent_process_pid( workspace: _AgentWorkspace, - attempt: Optional[EndpointAgentAttemptModel] = None, + agent_session: Optional[EndpointAgentSessionModel] = None, ) -> Optional[int]: - if attempt is not None and attempt.process_host not in (None, _get_process_host()): - return attempt.pid + state = _load_agent_process_state(workspace, agent_session=agent_session) + if state is None: + return None + if state.host not in (None, _get_process_host()): + return state.pid + if _is_process_group_running(state.pgid): + return state.pid + _clear_agent_process_state(workspace, state.pid) + return None + + +async def _abort_agent_session_process( + *, + agent_session: EndpointAgentSessionModel, + workspace: _AgentWorkspace, +) -> bool: + state = _load_agent_process_state(workspace, agent_session=agent_session) + if state is None: + return True + if state.host not in (None, _get_process_host()): + logger.info( + "Endpoint agent process %s for endpoint %s is running on host %s; " + "waiting for that host to abort it", + state.pid, + workspace.endpoint_name, + state.host, + ) + return False + if not _is_process_group_running(state.pgid): + _clear_agent_process_state(workspace, state.pid) + return True + + logger.info( + "Stopping endpoint agent process group %s for endpoint %s", + state.pgid, + workspace.endpoint_name, + ) + _write_trace_record( + workspace, + { + "type": "agent-process-abort-requested", + "pid": state.pid, + "pgid": state.pgid, + "host": state.host, + }, + ) + if not await _terminate_process_group(state.pgid): + return False + _clear_agent_process_state(workspace, state.pid) + return True + + +async def _terminate_process_group(pgid: int) -> bool: + if not _send_process_group_signal(pgid, signal.SIGTERM): + return True + await asyncio.sleep(_AGENT_PROCESS_ABORT_GRACE_SECONDS) + if not _is_process_group_running(pgid): + return True + _send_process_group_signal(pgid, signal.SIGKILL) + await asyncio.sleep(0) + return not _is_process_group_running(pgid) + + +def _send_process_group_signal(pgid: int, sig: signal.Signals) -> bool: + try: + os.killpg(pgid, sig) + except ProcessLookupError: + return False + except PermissionError: + logger.warning("Permission denied while signaling process group %s", pgid) + return True + return True + + +def _load_agent_process_state( + workspace: _AgentWorkspace, + agent_session: Optional[EndpointAgentSessionModel] = None, +) -> Optional[_AgentProcessState]: path = workspace.work_dir / _AGENT_PROCESS_STATE_NAME - if not path.exists(): - if attempt is None or attempt.pid is None: + if path.exists(): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + path.unlink(missing_ok=True) return None - if attempt.process_host not in (None, _get_process_host()): - return attempt.pid - return attempt.pid if _is_process_running(attempt.pid) else None - try: - data = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: + state = _parse_agent_process_state(data) + if state is not None: + return state path.unlink(missing_ok=True) + if agent_session is None or agent_session.pid is None: + return None + return _AgentProcessState( + pid=agent_session.pid, + pgid=agent_session.pid, + host=agent_session.process_host, + ) + + +def _parse_agent_process_state(data: Any) -> Optional[_AgentProcessState]: + if not isinstance(data, dict): return None - host = data.get("host") - if isinstance(host, str) and host and host != _get_process_host(): - pid = data.get("pid") - return pid if isinstance(pid, int) else None pid = data.get("pid") if not isinstance(pid, int): - path.unlink(missing_ok=True) return None - if _is_process_running(pid): - return pid - path.unlink(missing_ok=True) - return None + pgid = data.get("pgid", pid) + if not isinstance(pgid, int): + pgid = pid + host = data.get("host") + if not isinstance(host, str): + host = None + return _AgentProcessState(pid=pid, pgid=pgid, host=host) def _get_process_host() -> str: @@ -1592,6 +1930,48 @@ def _is_process_running(pid: int) -> bool: return True +def _is_process_group_running(pgid: int) -> bool: + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + except PermissionError: + live_process_found = _has_non_zombie_process_in_group(pgid) + return True if live_process_found is None else live_process_found + return True + + +def _has_non_zombie_process_in_group(pgid: int) -> Optional[bool]: + try: + result = subprocess.run( + ["ps", "-axo", "pgid=,stat="], + capture_output=True, + text=True, + timeout=2, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0: + return None + found_group = False + for line in result.stdout.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2: + continue + try: + line_pgid = int(parts[0]) + except ValueError: + continue + if line_pgid != pgid: + continue + found_group = True + stat = parts[1].strip() + if stat and not stat.startswith("Z"): + return True + return False if found_group else False + + def _write_trace_record(workspace: _AgentWorkspace, data: dict[str, Any]) -> None: if workspace.trace_path is None: return diff --git a/src/dstack/_internal/server/services/endpoints/agent/report.py b/src/dstack/_internal/server/services/endpoints/agent/report.py index ed872d6b4f..8fb10b8d08 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/report.py +++ b/src/dstack/_internal/server/services/endpoints/agent/report.py @@ -1,7 +1,7 @@ import uuid from typing import Optional -from pydantic import Field, root_validator +from pydantic import root_validator from dstack._internal.core.models.common import CoreModel @@ -12,8 +12,6 @@ "run_id": {"type": "string"}, "run_name": {"type": "string"}, "service_yaml": {"type": "string"}, - "recipe_sources": {"type": "array", "items": {"type": "string"}}, - "verification_summary": {"type": "string"}, "failure_summary": {"type": "string"}, }, "required": ["success"], @@ -28,8 +26,6 @@ class AgentFinalReport(CoreModel): run_id: Optional[uuid.UUID] = None run_name: Optional[str] = None service_yaml: Optional[str] = None - recipe_sources: list[str] = Field(default_factory=list) - verification_summary: Optional[str] = None failure_summary: Optional[str] = None @root_validator @@ -40,8 +36,6 @@ def _validate_report(cls, values: dict) -> dict: raise ValueError("successful agent report must include run_id") if not values.get("service_yaml"): raise ValueError("successful agent report must include service_yaml") - if not values.get("verification_summary"): - raise ValueError("successful agent report must include verification_summary") else: if not values.get("failure_summary"): raise ValueError("failed agent report must include failure_summary") diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md index 440cd5469c..0672485fd9 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md @@ -1,43 +1,199 @@ # Objective -You are the server-side endpoint deployment agent for dstack. Produce one dstack service that serves the requested model and prove it with a real model API request. +You are the endpoint deployment agent for dstack. Produce one final dstack +service for the requested model. Report success only after that service answers +a real request for the requested model through the dstack service URL. -Use the real `dstack` CLI and shell commands. Do not call hidden server APIs or helper functions such as `find_model_recipes`, `submit_service`, `get_run_status`, or `get_run_logs`. +Use the real `dstack` CLI and shell commands in this workspace. Load and follow +`/dstack` for dstack CLI/YAML syntax. Load and follow `/dstack-prototyping` for +how to test a model-serving recipe with tasks before verifying it as a service. +The skill files are installed under `.claude/skills` if you need to inspect +them directly. -Load and follow `/dstack` for CLI/config rules. Load and follow `/dstack-prototyping` for recipe research, hardware fit, experiments, debugging, and service finalization. +# Progress -Use primary sources when recipe or hardware behavior is uncertain. Save source URLs in `sources.jsonl` and include the relevant ones in `final_report.json`. +Write endpoint progress with: -Success requires a final dstack service run that answers a model API request for the requested model. Run status, service probes, and clean logs are evidence, not success. +```bash +progress "message for dstack endpoint logs" +``` -Write user-facing progress to `progress.jsonl` as single-line JSON objects: +The helper appends to `progress.jsonl`; the server shows only the message text. -```json -{"phase":"research","message":"Checking vLLM and SGLang support for Qwen/Qwen3-0.6B"} -``` +Write progress before your first investigation action, whenever you submit a +run, when new evidence changes the plan, when model verification succeeds or +fails, and before the final report. Messages should explain what you tried, what +happened, and what you will do next. Do not put raw YAML, command output, long +tables, traces, or secrets in progress. + +For `dstack endpoint logs`, write a short progress message for every meaningful +choice or action. Each message should say what you did or chose, why, what +evidence you used, what happened, and what you will do next. Include the run +name when a run is involved. Include the fleet or backend when a fleet or +backend is involved. Do not use generic phase labels as the message. + +# Workspace Files + +`submissions.jsonl` is append-only. For every dstack task or service you submit, +append one JSON line when you submit it and more JSON lines when you learn its +run id, status, URL, or final outcome. -Use progress only for milestones: research direction, experiment choice, candidate submitted, provisioning observation, verification result, cleanup, or terminal failure. Do not write YAML, command output, long tables, raw traces, or secrets to `progress.jsonl`. +Use these fields: -Record each dev environment, task, or service candidate in `candidates.jsonl`: +- `event`: `submit`, `update`, or `final` +- `name`: submitted dstack run name +- `type`: `task` or `service` +- `status`: current known status +- `config_path`: YAML file path, when applicable +- `run_id`: run id, when known +- `reason`: why this run exists or why its status changed +- `service_url`: service URL, when known + +Example: ```json -{"name":"qwen-test","type":"service","purpose":"final candidate","config_path":"qwen-test.dstack.yml","status":"submitted","run_id":null} +{"event":"submit","name":"qwen-endpoint-1","type":"task","status":"submitted","config_path":"qwen-endpoint-1.dstack.yml","reason":"test the serving image and local model request on an allowed fleet"} +{"event":"update","name":"qwen-endpoint-1","type":"task","status":"running","run_id":"..."} ``` -Update `status` when the candidate is ruled out, stopped, failed, or verified. Stop candidates you have ruled out unless they are needed for active debugging. +Do not rewrite previous `submissions.jsonl` lines; the latest line for a run is +the current record. Stop runs you no longer need unless they are still needed for +attach/SSH debugging, logs, or backend diagnosis. + +# Run Names + +Do not use the endpoint name itself as a submitted run name. + +Use only `-` for dstack runs submitted for +this endpoint. The first submitted run is `-1`, the second is +`-2`, and so on. Do not add framework, hardware, role, or purpose +suffixes to run names; record those details in workspace files and progress. + +# Endpoint Constraints + +Use existing allowed fleets only. Do not create, delete, apply, or edit fleets, +including `nodes`, `target`, `idle_duration`, backends, resources, max nodes, or +ownership. + +If the endpoint config lists fleets, use only those fleets. Otherwise, use the +existing project/imported fleets supplied in the request. + +The request also lists fixed endpoint constraints such as max price, spot +policy, backends, regions, instance types, fleets, env keys, tags, or backend +options. Do not submit a task or service that conflicts with those values. + +Put each fixed constraint that has a dstack service YAML field into the final +service YAML. For example, if the endpoint is limited to a fleet, max price, or +spot policy, the final `service_yaml` should include the corresponding `fleets`, +`max_price`, or `spot_policy` fields. Use `/dstack` for exact field names. + +If you cannot submit a useful task or service within the allowed fleets and +constraints, write a failed `final_report.json`. The `failure_summary` should +say which fleet or constraint blocked the run and what the user/admin would need +to change. + +# Task Usage + +Use `/dstack-prototyping` to learn how to use tasks. Keep in mind that using a +task is a must. This means `sleep infinity` and directly attaching inside the +task via SSH to run commands as the only way to use tasks. If there is any +ambiguity, follow `/dstack-prototyping` and do nothing that contradicts it. + +# Backend/Fleet Selection, Idle Duration, Instance Volumes + +Use only the fleets allowed by the endpoint request. If the endpoint request +also has constraints such as `backends`, `regions`, `instance_types`, +`spot_policy`, or `max_price`, apply them when running `dstack offer` if the CLI +has matching flags, and always apply them when submitting runs. + +## Backend And Fleet Choice + +Use `/dstack-prototyping` to learn how to select backends and fleets. + +Follow `/dstack-prototyping` skill on using a VM-based backend, Kubernetes backend, or SSH fleet that supports idle instances/instance volumes if there is such an option for the required GPU class. + +If backend allows (see above), use instance volumes to mount cache and model weights between runs. + +## Validating Offers + +When selecting backends/fleets and evaluating hardware that can be used to run +the model, use `dstack offer --json` and pass `--fleet` explicitly. If the +endpoint request has `fleets`, pass those fleets. Otherwise, pass every existing +fleet. If `--fleet` is not passed, `dstack offer` can show offers that are not +applicable to the fleets allowed by the endpoint request. Use these offers when +selecting fleet, backend, and hardware. + +Choosing fleet/backend is gated by classifying each against `https://dstack.ai/docs/concepts/backends.md`. VM-based backends are listed under `## VM-based` (they support idle instances and instance volumes). Kubernetes backend is listed under `## Container-based`, but supports instance volumes and thus is preferred over other container-based backends. +SSH fleets can be treated as VM-based backends as they support both idle instances (its equivalent) and instance volumes. + +## Submitting run + +When submitting a task or service, pass exact `fleets`, `backends`, and an +intentional `resources` range based on the choice made from offers, so the run +does not land outside the intended fleet/backend/hardware. + +## Endpoint Logs + +Endpoint logs should explain meaningful decisions and actions. + +When choosing fleet, backend, and hardware, write a log message that includes: + +- the selected fleet, backend, and resource range; +- the offer/docs evidence used for the choice; +- the viable alternatives not selected and the exact reason; +- the fleet, backend, and resources that will be used in the submitted YAML. +- how the selected and rejected fleets/backends were classified; + +Backend-choice log example: + +"I chose backend ... on fleet ... because ...; I did not choose ... because ...; +I will submit YAML with fleets=..., backends=..., resources=...." + +# Final Service + +The final `service_yaml` is what dstack will save as the endpoint recipe. It +must contain the full service config: `type: service`, the final run name, the +requested model, image/commands/port, resources, env references, and the fixed +endpoint constraints that apply to service YAML. + +Choose service resources from the least restrictive requirements supported by +the evidence, not from the exact machine that happened to run. For example, if +the model worked on an A40 but the evidence only says it needs an NVIDIA GPU +with at least 16GB memory, use that broader requirement. Use an exact GPU, +region, backend, or instance type only when the endpoint constraints require it +or the tested recipe depends on that exact choice. + +Use run status to know whether the final service is still starting, running, or +failed. Use logs to understand failures. When dstack exposes the service URL, +send a real model request through that URL. + +Do not write a successful `final_report.json` until the final service answers a +request for the requested model through the dstack service URL. + +# Secrets -Do not use the endpoint name itself as a candidate run name. Endpoint progress logs are keyed by endpoint name, and same-name service runs make debugging ambiguous. Use short attempt-style names for services, such as `-1`, `-2`, etc. +Do not print, cat, copy, or summarize `~/.dstack/config.yml`, tokens, secrets, +or environment variables. The dstack CLI is already configured; use it directly. +For final service verification, build absolute URLs from +`DSTACK_ENDPOINT_SERVER_URL` when needed and use +`DSTACK_ENDPOINT_BEARER_TOKEN` only as the bearer token in the request header. -Make each service YAML self-contained. If endpoint constraints map to service YAML fields, include them in the YAML; use CLI flags only as a checked override for preview/apply, not as the only record of the final service. +# Resume -Do not wait only for log text. During provisioning/startup, poll `dstack run get --json` and break on terminal statuses such as `failed` or `terminated`. Logs explain why a process failed; run JSON decides whether to keep waiting. +On startup or resume, inspect `final_report.json` and `submissions.jsonl` +before submitting anything new. If `final_report.json` already reports success +for the current endpoint configuration, do not submit another run. Otherwise use +the next run number and record why the old run is not enough. -Use normal service logs first. Do not use `dstack logs -d` for framework/application failures unless normal logs and run JSON are insufficient, because diagnostic logs may include runtime environment details that should not become endpoint trace material. +# Final Report -On startup or resume, inspect `agent_state.json`, `candidates.jsonl`, `commands.jsonl`, `verification.json`, and `final_report.json` before submitting anything new. Reuse an existing candidate only if it can still be verified. +On success, write `final_report.json`, then return the structured final report. -On success, write `verification.json` with the request URL, request model, response evidence, and final run name/id. Then write `final_report.json` and return the structured final report. +On failure, write `final_report.json` with a useful `failure_summary`, then +return the structured final report. -On failure, write `final_report.json` with the failure summary and the evidence needed for the next attempt. Then return the structured final report. +`final_report.json` must contain only the schema fields: `success`, `run_id`, +`run_name`, `service_yaml`, and `failure_summary`. -Stop after one correct service is verified. Do not benchmark, tune autoscaling, optimize hardware, or attempt P/D disaggregation unless the requested model cannot serve without it. +Stop after one correct service is verified. P/D disaggregation is not covered by +the current endpoint agent or `/dstack-prototyping` skill. diff --git a/src/dstack/_internal/server/services/endpoints/planning.py b/src/dstack/_internal/server/services/endpoints/planning.py index 3f8d145ef8..b40294bfdb 100644 --- a/src/dstack/_internal/server/services/endpoints/planning.py +++ b/src/dstack/_internal/server/services/endpoints/planning.py @@ -15,6 +15,7 @@ from dstack._internal.server.services.endpoints.names import get_endpoint_serving_run_name from dstack._internal.server.services.endpoints.presets import ( EndpointPreset, + EndpointPresetRecipe, EndpointPresetService, get_endpoint_preset_service, ) @@ -28,6 +29,7 @@ class EndpointPresetPlan: """A preset selected by endpoint planning and the run plan computed from it.""" preset: EndpointPreset + recipe: EndpointPresetRecipe run_plan: RunPlan @@ -84,40 +86,46 @@ async def find_preset_planning_result( user = await _ensure_user_has_ssh_key(session=session, user=user) first_unprovisionable_preset: Optional[EndpointPresetPlan] = None for preset in presets: - try: - run_spec = build_preset_run_spec( - endpoint_name=endpoint_name, - endpoint_configuration=endpoint_configuration, - preset=preset, - ) - _validate_run_spec_env_resolved(run_spec) - run_plan = await runs_services.get_plan( - session=session, - project=project, - user=user, - run_spec=run_spec, - max_offers=max_offers, - ) - except (ServerClientError, ValueError) as e: - logger.warning("Skipping endpoint preset %s: %s", preset.name, e) - continue - preset_plan = EndpointPresetPlan(preset=preset, run_plan=run_plan) - if _run_plan_has_available_offers(run_plan.job_plans): - return EndpointPresetPlanningResult( - provisionable=preset_plan, - unprovisionable=first_unprovisionable_preset, - ) - if first_unprovisionable_preset is None: - first_unprovisionable_preset = preset_plan + for recipe in preset.recipes: + try: + run_spec = build_preset_run_spec( + endpoint_name=endpoint_name, + endpoint_configuration=endpoint_configuration, + recipe=recipe, + ) + _validate_run_spec_env_resolved(run_spec) + run_plan = await runs_services.get_plan( + session=session, + project=project, + user=user, + run_spec=run_spec, + max_offers=max_offers, + ) + except (ServerClientError, ValueError) as e: + logger.warning( + "Skipping endpoint preset %s recipe %s: %s", + preset.model, + recipe.id, + e, + ) + continue + preset_plan = EndpointPresetPlan(preset=preset, recipe=recipe, run_plan=run_plan) + if _run_plan_has_available_offers(run_plan.job_plans): + return EndpointPresetPlanningResult( + provisionable=preset_plan, + unprovisionable=first_unprovisionable_preset, + ) + if first_unprovisionable_preset is None: + first_unprovisionable_preset = preset_plan return EndpointPresetPlanningResult(unprovisionable=first_unprovisionable_preset) def build_preset_service_configuration( endpoint_name: Optional[str], endpoint_configuration: EndpointConfiguration, - preset: EndpointPreset, + recipe: EndpointPresetRecipe, ) -> ServiceConfiguration: - service_configuration = preset.configuration.copy(deep=True) + service_configuration = recipe.service.copy(deep=True) service_configuration.name = get_endpoint_serving_run_name(endpoint_name) service_configuration.env.update(endpoint_configuration.env) for field in ProfileParams.__fields__: @@ -130,12 +138,12 @@ def build_preset_service_configuration( def build_preset_run_spec( endpoint_name: Optional[str], endpoint_configuration: EndpointConfiguration, - preset: EndpointPreset, + recipe: EndpointPresetRecipe, ) -> RunSpec: service_configuration = build_preset_service_configuration( endpoint_name=endpoint_name, endpoint_configuration=endpoint_configuration, - preset=preset, + recipe=recipe, ) return RunSpec( run_name=service_configuration.name, diff --git a/src/dstack/_internal/server/services/endpoints/preset_building.py b/src/dstack/_internal/server/services/endpoints/preset_building.py index 24fbff8040..b8c9be91fa 100644 --- a/src/dstack/_internal/server/services/endpoints/preset_building.py +++ b/src/dstack/_internal/server/services/endpoints/preset_building.py @@ -10,12 +10,16 @@ from dstack._internal.server.services import runs as runs_services from dstack._internal.server.services.endpoints.presets import ( EndpointPreset, - EndpointPresetReplicaSpecGroup, + EndpointPresetRecipe, + EndpointPresetValidation, + EndpointPresetValidationReplica, + make_endpoint_preset_recipe_id, + set_service_gpu_vendors_from_validations, ) from dstack._internal.utils.common import format_mib_as_gb -def build_endpoint_preset_from_run(name: str, run_model: RunModel) -> EndpointPreset: +def build_endpoint_preset_from_run(run_model: RunModel) -> EndpointPreset: run_spec = runs_services.get_run_spec(run_model) configuration = run_spec.configuration if not isinstance(configuration, ServiceConfiguration): @@ -24,30 +28,39 @@ def build_endpoint_preset_from_run(name: str, run_model: RunModel) -> EndpointPr raise ValueError("endpoint preset service must specify model") jobs_by_group = _get_current_registered_replica_jobs_by_group(run_model) - replica_spec_groups = [] + validation_replicas = [] for replica_group in configuration.replica_groups: group_name = replica_group.name if group_name is None: - raise ValueError("endpoint preset cannot be built from unnamed replica groups") + raise ValueError("endpoint preset cannot be built from unnamed service replicas") group_jobs = jobs_by_group.get(group_name, []) if not group_jobs: raise ValueError( f"endpoint preset cannot be built: replica group {group_name!r} " "has no registered running replicas" ) - replica_spec_groups.append( - EndpointPresetReplicaSpecGroup( - name=group_name, - resources=replica_group.resources, - tested_resources=[_get_job_resources_spec(job) for job in group_jobs], + validation_replicas.append( + EndpointPresetValidationReplica( + resources=[_get_job_resources_spec(job) for job in group_jobs], ) ) + service = configuration.copy(deep=True) + service.name = None + validation = EndpointPresetValidation(replicas=validation_replicas) + set_service_gpu_vendors_from_validations( + service=service, + validations=[validation], + ) return EndpointPreset( - name=name, model=configuration.model.name, - replica_spec_groups=replica_spec_groups, - configuration=configuration.copy(deep=True), + recipes=[ + EndpointPresetRecipe( + id=make_endpoint_preset_recipe_id(service), + service=service, + validations=[validation], + ) + ], ) diff --git a/src/dstack/_internal/server/services/endpoints/presets.py b/src/dstack/_internal/server/services/endpoints/presets.py index 55f7ce1c74..6a73d8023d 100644 --- a/src/dstack/_internal/server/services/endpoints/presets.py +++ b/src/dstack/_internal/server/services/endpoints/presets.py @@ -1,3 +1,4 @@ +import hashlib import json import os import re @@ -7,22 +8,19 @@ from pathlib import Path from typing import Any, NoReturn, Optional, Sequence +import gpuhunt import yaml from pydantic import ValidationError, parse_obj_as -from dstack._internal.core.models.common import CoreModel from dstack._internal.core.models.configurations import ( DEFAULT_REPLICA_GROUP_NAME, ServiceConfiguration, ) from dstack._internal.core.models.endpoint_presets import ( - EndpointPreset as EndpointPresetSummary, -) -from dstack._internal.core.models.endpoint_presets import ( - EndpointPresetDetails, -) -from dstack._internal.core.models.endpoint_presets import ( - EndpointPresetReplicaSpecGroup as EndpointPresetReplicaSpecGroupSummary, + EndpointPreset, + EndpointPresetRecipe, + EndpointPresetValidation, + EndpointPresetValidationReplica, ) from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.profiles import ProfileParams @@ -35,35 +33,17 @@ logger = get_logger(__name__) -class EndpointPresetReplicaSpecGroup(CoreModel): - """Ordered to match `ServiceConfiguration.replica_groups`; "0" is the implicit group.""" - - name: str - resources: ResourcesSpec - """Per-replica scheduling requirements used when applying the preset.""" - tested_resources: list[ResourcesSpec] - """Exact resources of the replicas that were running when the preset was verified.""" - - -class EndpointPreset(CoreModel): - name: str - model: str - replica_spec_groups: list[EndpointPresetReplicaSpecGroup] - """Sorted to match `configuration.replica_groups`; use group name "0" for the implicit group.""" - configuration: ServiceConfiguration - - class EndpointPresetService(ABC): @abstractmethod async def list_presets(self, project_name: str) -> list[EndpointPreset]: pass @abstractmethod - async def get_preset(self, project_name: str, name: str) -> Optional[EndpointPreset]: + async def get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: pass @abstractmethod - async def delete_preset(self, project_name: str, name: str) -> None: + async def delete_preset(self, project_name: str, model: str) -> None: pass @abstractmethod @@ -73,7 +53,7 @@ async def save_preset( preset: EndpointPreset, comments: Optional[Sequence[str]] = None, ) -> EndpointPreset: - """Store a prepared preset and return it with the storage-assigned name.""" + """Merge and store a model-level preset.""" pass @@ -84,11 +64,11 @@ def __init__(self, projects_dir: Path = settings.SERVER_PROJECTS_DIR_PATH) -> No async def list_presets(self, project_name: str) -> list[EndpointPreset]: return await run_async(self._list_presets, project_name) - async def get_preset(self, project_name: str, name: str) -> Optional[EndpointPreset]: - return await run_async(self._get_preset, project_name, name) + async def get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: + return await run_async(self._get_preset, project_name, model) - async def delete_preset(self, project_name: str, name: str) -> None: - return await run_async(self._delete_preset, project_name, name) + async def delete_preset(self, project_name: str, model: str) -> None: + return await run_async(self._delete_preset, project_name, model) async def save_preset( self, @@ -99,105 +79,117 @@ async def save_preset( return await run_async(self._save_preset, project_name, preset, comments or []) def _list_presets(self, project_name: str) -> list[EndpointPreset]: - presets_dir = self._get_project_presets_dir(project_name) - if not presets_dir.exists(): - return [] - presets = [] - for path in sorted(presets_dir.iterdir()): - if path.suffix not in [".yml", ".yaml"]: - continue - preset = self._load_preset(path) - if preset is not None: - presets.append(preset) - return presets + return [preset for preset, _ in self._list_merged_presets(project_name)] + + def _get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: + preset, _ = self._find_preset_by_model(project_name, model) + return preset + + def _delete_preset(self, project_name: str, model: str) -> None: + _, paths = self._find_preset_by_model(project_name, model) + if not paths: + raise FileNotFoundError(model) + for path in paths: + path.unlink() - def _get_preset(self, project_name: str, name: str) -> Optional[EndpointPreset]: + def _save_preset( + self, + project_name: str, + preset: EndpointPreset, + comments: Sequence[str], + ) -> EndpointPreset: + _validate_preset(preset) presets_dir = self._get_project_presets_dir(project_name) - if not presets_dir.exists(): - return None - for path in presets_dir.iterdir(): - if path.suffix not in [".yml", ".yaml"]: - continue - if _get_preset_name_from_path(path) == name: - return self._load_preset(path) - return None + presets_dir.mkdir(parents=True, exist_ok=True) + + existing_preset, existing_paths = self._find_preset_by_model(project_name, preset.model) + if existing_preset is not None: + saved_preset = _merge_presets(existing_preset, preset, prefer_incoming=True) + path = existing_paths[0] + else: + saved_preset = preset + path = self._get_available_preset_path(presets_dir, preset.model) + + data = _preset_to_data(saved_preset) + content = _format_preset_comments(comments) + yaml.safe_dump(data, sort_keys=False) + self._replace_preset_file(presets_dir=presets_dir, path=path, content=content) + for duplicate_path in existing_paths[1:]: + duplicate_path.unlink(missing_ok=True) + loaded_preset = self._load_preset(path) + if loaded_preset is None: + raise ValueError(f"saved endpoint preset {path} could not be loaded") + return loaded_preset - def _delete_preset(self, project_name: str, name: str) -> None: + def _find_preset_by_model( + self, + project_name: str, + model: str, + ) -> tuple[Optional[EndpointPreset], list[Path]]: + model_key = model.lower() + for preset, paths in self._list_merged_presets(project_name): + if preset.model.lower() == model_key: + return preset, paths + return None, [] + + def _list_merged_presets(self, project_name: str) -> list[tuple[EndpointPreset, list[Path]]]: presets_dir = self._get_project_presets_dir(project_name) if not presets_dir.exists(): - raise FileNotFoundError(name) - for path in presets_dir.iterdir(): - if path.suffix not in [".yml", ".yaml"]: + return [] + + presets_by_model: dict[str, EndpointPreset] = {} + paths_by_model: dict[str, list[Path]] = {} + model_order: list[str] = [] + for path in self._iter_preset_paths(presets_dir): + preset = self._load_preset(path) + if preset is None: continue - if _get_preset_name_from_path(path) == name: - path.unlink() - return - raise FileNotFoundError(name) + model_key = preset.model.lower() + existing = presets_by_model.get(model_key) + if existing is None: + presets_by_model[model_key] = preset + paths_by_model[model_key] = [path] + model_order.append(model_key) + continue + try: + presets_by_model[model_key] = _merge_presets(existing, preset) + except ValueError as e: + logger.warning("Skipping endpoint preset %s: %s", path, e) + continue + paths_by_model[model_key].append(path) + return [(presets_by_model[model], paths_by_model[model]) for model in model_order] + + def _iter_preset_paths(self, presets_dir: Path) -> list[Path]: + return [path for path in sorted(presets_dir.iterdir()) if path.suffix in [".yml", ".yaml"]] def _load_preset(self, path: Path) -> Optional[EndpointPreset]: try: with path.open("r") as f: data = yaml.safe_load(f) - if not isinstance(data, dict): - raise ValueError("preset must be a YAML object") - if data.get("type") != "endpoint-preset": - raise ValueError("preset must be an endpoint preset") - model = data.get("model") - if not isinstance(model, str) or model == "": - raise ValueError("preset must specify a model") - service_data = _get_service_data(data) - replica_spec_groups = _get_preset_replica_spec_groups(data, service_data) - service_data.setdefault("model", model) - service_data["type"] = "service" - configuration = ServiceConfiguration.parse_obj(service_data) - if configuration.model is None: - raise ValueError("preset service configuration must specify model") - if configuration.model.name.lower() != model.lower(): - raise ValueError("preset model must match the service model") - return EndpointPreset( - name=_get_preset_name_from_path(path), - model=model, - replica_spec_groups=replica_spec_groups, - configuration=configuration, - ) + preset = _preset_from_data(data) + _validate_preset(preset) + return preset except (OSError, ValidationError, ValueError, yaml.YAMLError) as e: logger.warning("Skipping endpoint preset %s: %s", path, e) return None - def _save_preset( - self, - project_name: str, - preset: EndpointPreset, - comments: Sequence[str], - ) -> EndpointPreset: - _validate_preset_before_save(preset) - presets_dir = self._get_project_presets_dir(project_name) - presets_dir.mkdir(parents=True, exist_ok=True) - data = _preset_to_data(preset) - content = _format_preset_comments(comments) + yaml.safe_dump(data, sort_keys=False) - base_name = _slugify_preset_name(preset.name) + def _get_project_presets_dir(self, project_name: str) -> Path: + return self._projects_dir / project_name / "presets" + + def _get_available_preset_path(self, presets_dir: Path, model: str) -> Path: + base_name = _slugify_model(model) suffix = 0 while True: name = base_name if suffix == 0 else f"{base_name}-{suffix + 1}" path = presets_dir / f"{name}.dstack.yml" - try: - self._write_preset_file(presets_dir=presets_dir, path=path, content=content) - break - except FileExistsError: - suffix += 1 - saved_preset = self._load_preset(path) - if saved_preset is None: - raise ValueError(f"saved endpoint preset {path} could not be loaded") - return saved_preset - - def _get_project_presets_dir(self, project_name: str) -> Path: - return self._projects_dir / project_name / "presets" + if not path.exists(): + return path + suffix += 1 - def _write_preset_file(self, presets_dir: Path, path: Path, content: str) -> None: + def _replace_preset_file(self, presets_dir: Path, path: Path, content: str) -> None: tmp_path = presets_dir / f".{path.name}.{uuid.uuid4().hex}.tmp" tmp_path.write_text(content, encoding="utf-8") try: - os.link(tmp_path, path) + os.replace(tmp_path, path) finally: tmp_path.unlink(missing_ok=True) @@ -209,72 +201,353 @@ def get_endpoint_preset_service() -> EndpointPresetService: return _endpoint_preset_service -def endpoint_preset_to_api_model(preset: EndpointPreset) -> EndpointPresetSummary: - return EndpointPresetSummary( - name=preset.name, - model=preset.model, - replica_spec_groups=[ - EndpointPresetReplicaSpecGroupSummary.parse_obj(group.dict()) - for group in preset.replica_spec_groups - ], - ) +def endpoint_preset_to_api_model(preset: EndpointPreset) -> EndpointPreset: + return EndpointPreset.parse_obj(preset.dict()) -def endpoint_preset_to_api_details(preset: EndpointPreset) -> EndpointPresetDetails: - return EndpointPresetDetails( - name=preset.name, - model=preset.model, - replica_spec_groups=[ - EndpointPresetReplicaSpecGroupSummary.parse_obj(group.dict()) - for group in preset.replica_spec_groups - ], - service=preset.configuration, - ) +def endpoint_preset_to_api_details(preset: EndpointPreset) -> EndpointPreset: + return EndpointPreset.parse_obj(preset.dict()) -def _get_preset_name_from_path(path: Path) -> str: - name = path.stem - if name.endswith(".dstack"): - name = name[: -len(".dstack")] - return name +def make_endpoint_preset_recipe_id(service: ServiceConfiguration) -> str: + service_data = _service_configuration_to_preset_data(service) + payload = json.dumps(service_data, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest()[:8] -def _validate_preset_before_save(preset: EndpointPreset) -> None: - expected_names = [ - group.name or DEFAULT_REPLICA_GROUP_NAME for group in preset.configuration.replica_groups - ] - _validate_replica_spec_groups( - replica_spec_groups=preset.replica_spec_groups, - expected_names=expected_names, +def _preset_from_data(data: Any) -> EndpointPreset: + if not isinstance(data, dict): + raise ValueError("preset must be a YAML object") + if data.get("type") != "endpoint-preset": + raise ValueError("preset must be an endpoint preset") + model = data.get("model") + if not isinstance(model, str) or model == "": + raise ValueError("preset must specify a model") + if "recipes" in data: + return EndpointPreset( + model=model, + recipes=_get_preset_recipes(data=data, model=model), + ) + return EndpointPreset( + model=model, + recipes=[_get_legacy_preset_recipe(data=data, model=model)], + ) + + +def _get_preset_recipes(data: dict[str, Any], model: str) -> list[EndpointPresetRecipe]: + raw_recipes = data.get("recipes") + if not isinstance(raw_recipes, list) or not raw_recipes: + raise ValueError("preset must specify non-empty recipes") + recipes = [_get_preset_recipe(raw_recipe, model=model) for raw_recipe in raw_recipes] + recipe_ids = [recipe.id for recipe in recipes] + if len(recipe_ids) != len(set(recipe_ids)): + raise ValueError("preset recipes must have unique ids") + return recipes + + +def _get_preset_recipe(raw_recipe: Any, model: str) -> EndpointPresetRecipe: + if not isinstance(raw_recipe, dict): + raise ValueError("preset recipes must be objects") + recipe_id = raw_recipe.get("id") + if not isinstance(recipe_id, str) or recipe_id == "": + raise ValueError("preset recipe must specify id") + service_data = _get_service_data(raw_recipe, model=model) + service = ServiceConfiguration.parse_obj(service_data) + validations = _get_validations(raw_recipe, service=service) + return EndpointPresetRecipe(id=recipe_id, service=service, validations=validations) + + +def _get_legacy_preset_recipe(data: dict[str, Any], model: str) -> EndpointPresetRecipe: + service_data = _get_service_data(data, model=model, allow_missing_resources=True) + raw_groups = _get_legacy_replica_spec_groups(data=data, service_data=service_data) + _apply_legacy_replica_group_resources(service_data=service_data, groups=raw_groups) + service = ServiceConfiguration.parse_obj(service_data) + validation = EndpointPresetValidation( + replicas=[ + EndpointPresetValidationReplica( + resources=[ + ResourcesSpec.parse_obj(resources) + for resources in _get_legacy_group_tested_resources(group) + ] + ) + for group in raw_groups + ] + ) + return EndpointPresetRecipe( + id=make_endpoint_preset_recipe_id(service), + service=service, + validations=[validation], ) -def _get_service_data(data: dict[str, Any]) -> dict[str, Any]: +def _get_service_data( + data: dict[str, Any], + model: str, + allow_missing_resources: bool = False, +) -> dict[str, Any]: service_data = data.get("service") if not isinstance(service_data, dict): - raise ValueError("preset must specify a service object") + raise ValueError("preset recipe must specify a service object") + service_data = deepcopy(service_data) if service_data.get("type") not in (None, "service"): raise ValueError("preset service object must be a service configuration") if "name" in service_data: raise ValueError("preset service object must not specify name") - if "resources" in service_data: - raise ValueError("preset service object must not specify resources") profile_fields = sorted(set(ProfileParams.__fields__) & set(service_data)) if profile_fields: raise ValueError( "preset service object must not specify profile fields: " + ", ".join(profile_fields) ) - _validate_service_replica_groups(service_data) - return deepcopy(service_data) + if not allow_missing_resources: + _validate_service_has_resources(service_data) + service_data.setdefault("model", model) + service_data["type"] = "service" + configuration = ServiceConfiguration.parse_obj(service_data) + if configuration.model is None: + raise ValueError("preset service configuration must specify model") + if configuration.model.name.lower() != model.lower(): + raise ValueError("preset model must match the service model") + return service_data + + +def _validate_service_has_resources(service_data: dict[str, Any]) -> None: + replicas = service_data.get("replicas") + if isinstance(replicas, list): + for group in replicas: + if not isinstance(group, dict): + raise ValueError("preset service replica groups must be objects") + if "resources" not in group: + raise ValueError("preset service replica groups must specify resources") + return + if "resources" not in service_data: + raise ValueError("preset service object must specify resources") + + +def _get_validations( + raw_recipe: dict[str, Any], + service: ServiceConfiguration, +) -> list[EndpointPresetValidation]: + raw_validations = raw_recipe.get("validations") + if not isinstance(raw_validations, list) or not raw_validations: + raise ValueError("preset recipe must specify non-empty validations") + validations = [ + EndpointPresetValidation.parse_obj(raw_validation) for raw_validation in raw_validations + ] + _validate_validations(validations=validations, service=service) + return validations + + +def _validate_preset(preset: EndpointPreset) -> None: + if not preset.recipes: + raise ValueError("preset must specify non-empty recipes") + recipe_ids = [recipe.id for recipe in preset.recipes] + if len(recipe_ids) != len(set(recipe_ids)): + raise ValueError("preset recipes must have unique ids") + for recipe in preset.recipes: + if recipe.service.model is None: + raise ValueError("preset recipe service must specify model") + if recipe.service.model.name.lower() != preset.model.lower(): + raise ValueError("preset model must match the recipe service model") + _validate_validations(validations=recipe.validations, service=recipe.service) + _validate_service_gpu_vendor_matches_validations( + service=recipe.service, + validations=recipe.validations, + ) + + +def set_service_gpu_vendors_from_validations( + service: ServiceConfiguration, + validations: list[EndpointPresetValidation], +) -> None: + for group_num, group in enumerate(service.replica_groups): + resources = group.resources + if resources is None or not _requires_gpu(resources): + continue + validation_vendor = _get_validation_group_gpu_vendor( + validations=validations, + group_num=group_num, + ) + if validation_vendor is None: + continue + if resources.gpu is None: + continue + if resources.gpu.vendor is not None and resources.gpu.vendor != validation_vendor: + raise ValueError("preset service GPU vendor does not match validation") + group_resources = _get_service_group_resources(service, group_num) + if group_resources.gpu is None: + continue + group_resources.gpu.vendor = validation_vendor + + +def _validate_service_gpu_vendor_matches_validations( + service: ServiceConfiguration, + validations: list[EndpointPresetValidation], +) -> None: + for group_num, group in enumerate(service.replica_groups): + resources = group.resources + if resources is None or not _requires_gpu(resources): + continue + if resources.gpu is None or resources.gpu.vendor is None: + continue + validation_vendor = _get_validation_group_gpu_vendor( + validations=validations, + group_num=group_num, + ) + if validation_vendor is not None and resources.gpu.vendor != validation_vendor: + raise ValueError("preset service GPU vendor does not match validation") + + +def _get_validation_group_gpu_vendor( + validations: list[EndpointPresetValidation], + group_num: int, +): + vendors = set() + for validation in validations: + for resources in validation.replicas[group_num].resources: + if resources.gpu is None or resources.gpu.count.min == 0: + continue + vendor = _get_validation_resources_gpu_vendor(resources) + if vendor is not None: + vendors.add(vendor) + if len(vendors) > 1: + raise ValueError("preset validations must not mix GPU vendors in a replica group") + return next(iter(vendors), None) + + +def _get_validation_resources_gpu_vendor(resources: ResourcesSpec): + gpu = resources.gpu + if gpu is None: + return None + if gpu.vendor is not None: + return gpu.vendor + if not gpu.name: + return None + vendors = {_get_gpu_name_vendor(name) for name in gpu.name} + vendors.discard(None) + if len(vendors) > 1: + raise ValueError("preset validations must not mix GPU vendors in a replica group") + return next(iter(vendors), None) + + +def _get_gpu_name_vendor(name: str): + if _is_known_gpu_name(name, gpuhunt.KNOWN_NVIDIA_GPUS): + return gpuhunt.AcceleratorVendor.NVIDIA + if _is_known_gpu_name(name, gpuhunt.KNOWN_AMD_GPUS): + return gpuhunt.AcceleratorVendor.AMD + if _is_known_gpu_name(name, gpuhunt.KNOWN_INTEL_ACCELERATORS): + return gpuhunt.AcceleratorVendor.INTEL + if _is_known_gpu_name(name, gpuhunt.KNOWN_TENSTORRENT_ACCELERATORS): + return gpuhunt.AcceleratorVendor.TENSTORRENT + if name.startswith("tpu-"): + return gpuhunt.AcceleratorVendor.GOOGLE + return None + + +def _is_known_gpu_name(name: str, known_gpus: list) -> bool: + return any(gpu.name.lower() == name.lower() for gpu in known_gpus) + + +def _get_service_group_resources( + service: ServiceConfiguration, + group_num: int, +) -> ResourcesSpec: + if isinstance(service.replicas, list): + resources = service.replicas[group_num].resources + else: + resources = service.resources + if resources is None: + raise ValueError("preset service object must specify resources") + return resources + + +def _requires_gpu(resources: ResourcesSpec) -> bool: + gpu = resources.gpu + if gpu is None: + return False + if gpu.count.max == 0: + return False + return gpu.count.min != 0 or gpu.count.max is not None + + +def _validate_validations( + validations: list[EndpointPresetValidation], + service: ServiceConfiguration, +) -> None: + expected_group_count = len(service.replica_groups) + for validation in validations: + if len(validation.replicas) != expected_group_count: + raise ValueError("preset validation replicas must match service replica group order") + for group in validation.replicas: + if not group.resources: + raise ValueError("preset validation replicas must specify resources") + for resources in group.resources: + _validate_replica_resources_are_exact(resources) + + +def _merge_presets( + existing: EndpointPreset, + incoming: EndpointPreset, + prefer_incoming: bool = False, +) -> EndpointPreset: + if existing.model.lower() != incoming.model.lower(): + raise ValueError("cannot merge endpoint presets for different models") + recipes = [EndpointPresetRecipe.parse_obj(recipe.dict()) for recipe in existing.recipes] + recipes_by_id = {recipe.id: recipe for recipe in recipes} + incoming_recipe_ids = [] + for incoming_recipe in incoming.recipes: + existing_recipe = recipes_by_id.get(incoming_recipe.id) + if existing_recipe is None: + recipe = EndpointPresetRecipe.parse_obj(incoming_recipe.dict()) + recipes.append(recipe) + recipes_by_id[recipe.id] = recipe + incoming_recipe_ids.append(recipe.id) + continue + if _service_configuration_to_preset_data(existing_recipe.service) != ( + _service_configuration_to_preset_data(incoming_recipe.service) + ): + raise ValueError(f"endpoint preset recipe id conflict: {incoming_recipe.id}") + _merge_recipe_validations(existing_recipe, incoming_recipe.validations) + incoming_recipe_ids.append(existing_recipe.id) + if prefer_incoming: + preferred_ids = set(incoming_recipe_ids) + recipes = [recipes_by_id[recipe_id] for recipe_id in incoming_recipe_ids] + [ + recipe for recipe in recipes if recipe.id not in preferred_ids + ] + return EndpointPreset(model=existing.model, recipes=recipes) + + +def _merge_recipe_validations( + recipe: EndpointPresetRecipe, + incoming_validations: list[EndpointPresetValidation], +) -> None: + existing_keys = {_validation_key(validation) for validation in recipe.validations} + for validation in incoming_validations: + key = _validation_key(validation) + if key in existing_keys: + continue + recipe.validations.append(EndpointPresetValidation.parse_obj(validation.dict())) + existing_keys.add(key) + + +def _validation_key(validation: EndpointPresetValidation) -> str: + data = json.loads(validation.json(exclude_none=True)) + return json.dumps(data, sort_keys=True, separators=(",", ":")) def _preset_to_data(preset: EndpointPreset) -> dict[str, Any]: return { "type": "endpoint-preset", "model": preset.model, - "service": _service_configuration_to_preset_data(preset.configuration), - "replica_spec_groups": [ - _replica_spec_group_to_data(group) for group in preset.replica_spec_groups + "recipes": [_recipe_to_data(recipe) for recipe in preset.recipes], + } + + +def _recipe_to_data(recipe: EndpointPresetRecipe) -> dict[str, Any]: + return { + "id": recipe.id, + "service": _service_configuration_to_preset_data(recipe.service), + "validations": [ + json.loads(validation.json(exclude_none=True)) for validation in recipe.validations ], } @@ -285,7 +558,6 @@ def _service_configuration_to_preset_data( service_data = json.loads(configuration.json(exclude_none=True)) service_data.pop("type", None) service_data.pop("name", None) - service_data.pop("resources", None) for field in ProfileParams.__fields__: service_data.pop(field, None) if configuration.env: @@ -298,24 +570,9 @@ def _service_configuration_to_preset_data( for field, value in list(service_data.items()): if value in ({}, []): service_data.pop(field) - replicas = service_data.get("replicas") - if isinstance(replicas, list): - for group in replicas: - if isinstance(group, dict): - group.pop("resources", None) return service_data -def _replica_spec_group_to_data(group: EndpointPresetReplicaSpecGroup) -> dict[str, Any]: - return { - "name": group.name, - "resources": json.loads(group.resources.json(exclude_none=True)), - "tested_resources": [ - json.loads(resources.json(exclude_none=True)) for resources in group.tested_resources - ], - } - - def _env_item_to_preset_data(key: str, value: str | EnvSentinel) -> str: if isinstance(value, EnvSentinel) or _is_secret_like_env(key=key, value=value): return key @@ -338,69 +595,44 @@ def _format_preset_comments(comments: Sequence[str]) -> str: return "\n".join(lines) + "\n" -def _slugify_preset_name(name: str) -> str: - slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") +def _slugify_model(model: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", model.lower()).strip("-") return slug or "endpoint-preset" -def _validate_service_replica_groups(service_data: dict[str, Any]) -> None: - replicas = service_data.get("replicas") - if not isinstance(replicas, list): - return - for group in replicas: - if not isinstance(group, dict): - raise ValueError("preset service replica groups must be objects") - if "resources" in group: - raise ValueError("preset service replica groups must not specify resources") - - -def _get_preset_replica_spec_groups( +def _get_legacy_replica_spec_groups( + *, data: dict[str, Any], service_data: dict[str, Any], -) -> list[EndpointPresetReplicaSpecGroup]: - if "resources" in data or "replica_resources" in data: - raise ValueError("preset must specify replica_spec_groups") - raw_replica_spec_groups = data.get("replica_spec_groups") - if not isinstance(raw_replica_spec_groups, list) or not raw_replica_spec_groups: - raise ValueError("preset must specify non-empty replica_spec_groups") - if not all(isinstance(group, dict) for group in raw_replica_spec_groups): - raise ValueError("preset replica_spec_groups must be a list of objects") - raw_replica_spec_groups = [dict(group) for group in raw_replica_spec_groups] +) -> list[dict[str, Any]]: + raw_groups = data.get("replica_spec_groups") + if not isinstance(raw_groups, list) or not raw_groups: + raise ValueError("legacy preset must specify non-empty replica_spec_groups") + if not all(isinstance(group, dict) for group in raw_groups): + raise ValueError("legacy preset replica_spec_groups must be objects") + groups = [_normalize_legacy_replica_spec_group(dict(group)) for group in raw_groups] expected_names = _get_expected_replica_group_names(service_data) - if expected_names == [DEFAULT_REPLICA_GROUP_NAME] and "name" not in raw_replica_spec_groups[0]: - raw_replica_spec_groups[0]["name"] = DEFAULT_REPLICA_GROUP_NAME - raw_replica_spec_groups = [ - _normalize_replica_spec_group(group) for group in raw_replica_spec_groups - ] - replica_spec_groups = [ - EndpointPresetReplicaSpecGroup.parse_obj(group) for group in raw_replica_spec_groups - ] - _validate_replica_spec_groups( - replica_spec_groups=replica_spec_groups, expected_names=expected_names - ) - _apply_replica_spec_group_resources( - service_data=service_data, - replica_spec_groups=replica_spec_groups, - ) - return replica_spec_groups + if expected_names == [DEFAULT_REPLICA_GROUP_NAME] and "name" not in groups[0]: + groups[0]["name"] = DEFAULT_REPLICA_GROUP_NAME + group_names = [group.get("name") for group in groups] + if group_names != expected_names: + raise ValueError( + "legacy preset replica_spec_groups must match replica group order: " + + ", ".join(expected_names) + ) + return groups -def _normalize_replica_spec_group(group: dict[str, Any]) -> dict[str, Any]: +def _normalize_legacy_replica_spec_group(group: dict[str, Any]) -> dict[str, Any]: if "resources" in group or "tested_resources" in group: if "resources" not in group or "tested_resources" not in group: raise ValueError( - "preset replica_spec_groups must specify resources and tested_resources" - ) - if "replica_specs" in group: - raise ValueError( - "preset replica_spec_groups must not mix replica_specs with resources" + "legacy preset replica_spec_groups must specify resources and tested_resources" ) return group - replica_specs = group.get("replica_specs") if not isinstance(replica_specs, list) or not replica_specs: - raise ValueError("preset replica_spec_groups must specify non-empty tested_resources") - group = dict(group) + raise ValueError("legacy preset replica_spec_groups must specify resources") group["resources"] = replica_specs[0] group["tested_resources"] = replica_specs group.pop("replica_specs", None) @@ -411,35 +643,36 @@ def _get_expected_replica_group_names(service_data: dict[str, Any]) -> list[str] replicas = service_data.get("replicas") if not isinstance(replicas, list): return [DEFAULT_REPLICA_GROUP_NAME] - return [_get_replica_group_name(group) for group in replicas] + names = [] + for group in replicas: + if not isinstance(group, dict): + raise ValueError("preset service replica groups must be objects") + name = group.get("name") + if not isinstance(name, str) or name == "": + raise ValueError("preset service replica groups must specify names") + names.append(name) + return names -def _get_replica_group_name(replica_group: Any) -> str: - if not isinstance(replica_group, dict): - raise ValueError("preset service replica groups must be objects") - name = replica_group.get("name") - if not isinstance(name, str) or name == "": - raise ValueError( - "preset service replica groups must specify names when using replica_spec_groups" - ) - return name +def _apply_legacy_replica_group_resources( + *, + service_data: dict[str, Any], + groups: list[dict[str, Any]], +) -> None: + replicas = service_data.get("replicas") + if not isinstance(replicas, list): + service_data["resources"] = groups[0]["resources"] + return + resources_by_group = {group["name"]: group["resources"] for group in groups} + for group in replicas: + group["resources"] = resources_by_group[group["name"]] -def _validate_replica_spec_groups( - replica_spec_groups: list[EndpointPresetReplicaSpecGroup], - expected_names: list[str], -) -> None: - group_names = [group.name for group in replica_spec_groups] - if group_names != expected_names: - raise ValueError( - "preset replica_spec_groups must match replica group order: " - + ", ".join(expected_names) - ) - for group in replica_spec_groups: - if not group.tested_resources: - raise ValueError("preset replica_spec_groups must specify non-empty tested_resources") - for resources in group.tested_resources: - _validate_replica_resources_are_exact(resources) +def _get_legacy_group_tested_resources(group: dict[str, Any]) -> list[Any]: + tested_resources = group.get("tested_resources") + if not isinstance(tested_resources, list) or not tested_resources: + raise ValueError("legacy preset group must specify non-empty tested_resources") + return tested_resources def _validate_replica_resources_are_exact(resources: ResourcesSpec) -> None: @@ -465,7 +698,7 @@ def _validate_replica_resources_are_exact(resources: ResourcesSpec) -> None: def _raise_loose_replica_resources() -> NoReturn: - raise ValueError("preset tested_resources must use exact replica resources") + raise ValueError("preset validations must use exact replica resources") def _is_exact_range(value) -> bool: @@ -475,16 +708,3 @@ def _is_exact_range(value) -> bool: and value.max is not None and value.min == value.max ) - - -def _apply_replica_spec_group_resources( - service_data: dict[str, Any], - replica_spec_groups: list[EndpointPresetReplicaSpecGroup], -) -> None: - replicas = service_data.get("replicas") - if not isinstance(replicas, list): - service_data["resources"] = replica_spec_groups[0].resources.dict() - return - resources_by_group = {group.name: group.resources.dict() for group in replica_spec_groups} - for group in replicas: - group["resources"] = resources_by_group[group["name"]] diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index b001a0d3ab..4e6f95c4e5 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -13,13 +13,6 @@ logger = get_logger(__name__) -def _parse_positive_float(value: str) -> float: - parsed = float(value) - if parsed <= 0: - raise ValueError("value must be greater than 0") - return parsed - - def _getenv_non_empty(name: str) -> str | None: value = os.getenv(name) if value is None or value == "": @@ -41,9 +34,6 @@ def _getenv_non_empty(name: str) -> str | None: AGENT_ANTHROPIC_API_KEY = _getenv_non_empty("DSTACK_AGENT_ANTHROPIC_API_KEY") AGENT_CLAUDE_PATH = _getenv_non_empty("DSTACK_AGENT_CLAUDE_PATH") AGENT_ANTHROPIC_MODEL = os.getenv("DSTACK_AGENT_ANTHROPIC_MODEL", "claude-opus-4-8") -AGENT_ANTHROPIC_MAX_BUDGET = environ.get_callback( - "DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD", _parse_positive_float -) DATABASE_URL = os.getenv( "DSTACK_DATABASE_URL", f"sqlite+aiosqlite:///{str(SERVER_DATA_DIR_PATH.absolute())}/sqlite.db" diff --git a/src/dstack/api/server/_endpoint_presets.py b/src/dstack/api/server/_endpoint_presets.py index 09ff0b08c7..9542cdce64 100644 --- a/src/dstack/api/server/_endpoint_presets.py +++ b/src/dstack/api/server/_endpoint_presets.py @@ -2,7 +2,7 @@ from pydantic import parse_obj_as -from dstack._internal.core.models.endpoint_presets import EndpointPreset, EndpointPresetDetails +from dstack._internal.core.models.endpoint_presets import EndpointPreset from dstack._internal.server.schemas.endpoint_presets import ( DeleteEndpointPresetsRequest, GetEndpointPresetRequest, @@ -15,16 +15,16 @@ def list(self, project_name: str) -> List[EndpointPreset]: resp = self._request(f"/api/project/{project_name}/endpoints/presets/list") return parse_obj_as(List[EndpointPreset.__response__], resp.json()) - def get(self, project_name: str, name: str) -> EndpointPresetDetails: - body = GetEndpointPresetRequest(name=name) + def get(self, project_name: str, model: str) -> EndpointPreset: + body = GetEndpointPresetRequest(model=model) resp = self._request( f"/api/project/{project_name}/endpoints/presets/get", body=body.json(), ) - return parse_obj_as(EndpointPresetDetails.__response__, resp.json()) + return parse_obj_as(EndpointPreset.__response__, resp.json()) - def delete(self, project_name: str, names: List[str]) -> None: - body = DeleteEndpointPresetsRequest(names=names) + def delete(self, project_name: str, models: List[str]) -> None: + body = DeleteEndpointPresetsRequest(models=models) self._request( f"/api/project/{project_name}/endpoints/presets/delete", body=body.json(), diff --git a/src/tests/_internal/cli/commands/test_endpoint.py b/src/tests/_internal/cli/commands/test_endpoint.py index f886db79dc..07de32e142 100644 --- a/src/tests/_internal/cli/commands/test_endpoint.py +++ b/src/tests/_internal/cli/commands/test_endpoint.py @@ -43,11 +43,14 @@ def stop(self, project_name, names): class _FakeLogs: - def __init__(self): + def __init__(self, responses=None): + self._responses = responses self.requests = [] def poll(self, project_name, body): self.requests.append(body) + if self._responses is not None: + return self._responses.pop(0) return JobSubmissionLogs( logs=[ LogEvent( @@ -59,30 +62,78 @@ def poll(self, project_name, body): ) -def _get_command(endpoint: Endpoint) -> EndpointCommand: +def _get_command(endpoint: Endpoint, logs=None) -> EndpointCommand: command = EndpointCommand.__new__(EndpointCommand) command.api = SimpleNamespace( project="main", client=SimpleNamespace( endpoints=_FakeEndpoints(endpoint), - logs=_FakeLogs(), + logs=logs or _FakeLogs(), ), ) return command class TestEndpointCommand: + def test_preset_list_verbose_can_be_before_or_after_list_action(self): + parser = argparse.ArgumentParser() + EndpointCommand(parser)._register() + + assert parser.parse_args(["preset", "-v", "list"]).verbose is True + assert parser.parse_args(["preset", "list", "-v"]).verbose is True + assert parser.parse_args(["preset", "list"]).verbose is False + def test_reads_endpoint_logs(self): endpoint = _get_endpoint() command = _get_command(endpoint) logs = list(command._get_endpoint_logs(endpoint=endpoint, start_time=None)) - assert logs == [b"agent log\n"] + assert len(logs) == 1 + assert logs[0].startswith(b"[") + assert logs[0].endswith(b"] agent log\n") request = command.api.client.logs.requests[0] assert request.run_name == endpoint.name assert request.job_submission_id == endpoint.id + def test_watch_endpoint_logs_does_not_swallow_same_timestamp_logs(self, monkeypatch): + endpoint = _get_endpoint() + timestamp = datetime.now(timezone.utc) + logs_api = _FakeLogs( + responses=[ + JobSubmissionLogs( + logs=[ + LogEvent( + timestamp=timestamp, + log_source=LogEventSource.STDOUT, + message=base64.b64encode(b"first\n").decode(), + ), + ], + ), + JobSubmissionLogs( + logs=[ + LogEvent( + timestamp=timestamp, + log_source=LogEventSource.STDOUT, + message=base64.b64encode(b"first\n").decode(), + ), + LogEvent( + timestamp=timestamp, + log_source=LogEventSource.STDOUT, + message=base64.b64encode(b"second\n").decode(), + ), + ], + ), + ], + ) + command = _get_command(endpoint, logs=logs_api) + monkeypatch.setattr("dstack._internal.cli.commands.endpoint.time.sleep", lambda _: None) + + logs = command._get_endpoint_logs(endpoint=endpoint, start_time=None, watch=True) + + assert next(logs).endswith(b"] first\n") + assert next(logs).endswith(b"] second\n") + def test_stop_endpoint(self): endpoint = _get_endpoint() command = _get_command(endpoint) diff --git a/src/tests/_internal/cli/services/configurators/test_endpoint.py b/src/tests/_internal/cli/services/configurators/test_endpoint.py index 2e0d34560f..1b696f1944 100644 --- a/src/tests/_internal/cli/services/configurators/test_endpoint.py +++ b/src/tests/_internal/cli/services/configurators/test_endpoint.py @@ -20,7 +20,6 @@ EndpointConfiguration, EndpointPlan, EndpointPlanJobOffers, - EndpointPlanReplicaSpecGroup, EndpointPresetPolicy, EndpointProvisioningPlanAgent, EndpointProvisioningPlanNone, @@ -119,15 +118,9 @@ def test_prints_preset_without_resources_property(self, monkeypatch: pytest.Monk resources = ResourcesSpec.parse_obj({"gpu": "16GB", "disk": "60GB"}) plan = _get_endpoint_plan( EndpointProvisioningPlanPreset( - preset_name="qwen-vllm", + preset_model="Qwen/Qwen3-0.6B", + recipe_id="vllm-a40", service_name="qwen-endpoint-serving", - replica_spec_groups=[ - EndpointPlanReplicaSpecGroup( - name="0", - resources=resources, - tested_resources=[resources], - ) - ], job_offers=[ EndpointPlanJobOffers( replica_group="0", @@ -146,7 +139,8 @@ def test_prints_preset_without_resources_property(self, monkeypatch: pytest.Monk output = console.export_text() assert "Resources" not in output - assert "Preset qwen-vllm" in output + assert "Preset Qwen/Qwen3-0.6B" in output + assert "Recipe vllm-a40" in output def test_prints_agent_reason_when_preset_has_no_offers(self, monkeypatch: pytest.MonkeyPatch): console = _patch_console(monkeypatch) @@ -164,19 +158,6 @@ def test_prints_agent_reason_when_preset_has_no_offers(self, monkeypatch: pytest assert "Endpoint preset qwen matched but has no available offers." in output assert "Agent" not in output - def test_prints_agent_budget_when_set(self, monkeypatch: pytest.MonkeyPatch): - console = _patch_console(monkeypatch) - plan = _get_endpoint_plan( - EndpointProvisioningPlanAgent(agent_model="test-agent", max_budget=2.0) - ) - - _print_endpoint_plan(plan) - - output = console.export_text() - assert "Agent budget" in output - assert "$2" in output - assert "test-agent" not in output - class TestPrintSubmittedEndpointMessage: def test_prints_generic_detach_message(self, monkeypatch: pytest.MonkeyPatch): diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py index b0d8a919fe..05e089254b 100644 --- a/src/tests/_internal/cli/utils/test_endpoint.py +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -40,10 +40,18 @@ def test_default_table_does_not_show_status_message(self): assert "ERROR" not in [column.header for column in table.columns] + def test_default_table_shows_preset_policy_not_service_run(self): + table = get_endpoints_table([_get_endpoint()]) + + headers = [column.header for column in table.columns] + assert "POLICY" in headers + assert "SERVICE RUN" not in headers + def test_verbose_table_shows_status_message(self): table = get_endpoints_table([_get_endpoint()], verbose=True) assert "ERROR" in [column.header for column in table.columns] + assert "SERVICE RUN" in [column.header for column in table.columns] def test_failed_status_without_reason_is_colored(self): table = get_endpoints_table([_get_endpoint(status_message=None)]) @@ -89,11 +97,7 @@ def test_no_offers_status_is_colored(self): def test_agent_failed_status_is_colored(self): table = get_endpoints_table( - [ - _get_endpoint( - status_message="Server agent process exited without a verification report" - ) - ] + [_get_endpoint(status_message="Server agent process exited without a final report")] ) status_column = next(column for column in table.columns if column.header == "STATUS") @@ -105,11 +109,11 @@ def test_running_status_is_colored(self): status_column = next(column for column in table.columns if column.header == "STATUS") assert status_column._cells == ["[bold sea_green3]running[/]"] - def test_clauding_status_is_colored(self): - table = get_endpoints_table([_get_endpoint(status=EndpointStatus.CLAUDING)]) + def test_prototyping_status_is_colored(self): + table = get_endpoints_table([_get_endpoint(status=EndpointStatus.PROTOTYPING)]) status_column = next(column for column in table.columns if column.header == "STATUS") - assert status_column._cells == ["[bold medium_purple1]clauding[/]"] + assert status_column._cells == ["[bold medium_purple1]prototyping[/]"] def test_stopping_status_is_colored(self): table = get_endpoints_table([_get_endpoint(status=EndpointStatus.STOPPING)]) @@ -138,7 +142,8 @@ def test_shows_endpoint_details(self): "[bold]Endpoint[/bold]", "[bold]Model[/bold]", "[bold]Status[/bold]", - "[bold]Run[/bold]", + "[bold]Preset policy[/bold]", + "[bold]Service run[/bold]", "[bold]URL[/bold]", "[bold]Created[/bold]", "[bold]Error[/bold]", @@ -149,6 +154,7 @@ def test_shows_endpoint_details(self): "qwen-endpoint", "Qwen/Qwen3-0.6B", "[indian_red1]no preset[/]", + "reuse-or-create", "-", "-", "now", @@ -157,7 +163,7 @@ def test_shows_endpoint_details(self): class TestFilterEndpointsForListing: - def test_default_shows_unfinished_and_latest_finished(self): + def test_default_shows_unfinished_only_when_present(self): endpoints = [ _get_endpoint( name="failed-old", @@ -175,8 +181,8 @@ def test_default_shows_unfinished_and_latest_finished(self): created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), ), _get_endpoint( - name="clauding", - status=EndpointStatus.CLAUDING, + name="prototyping", + status=EndpointStatus.PROTOTYPING, created_at=datetime(2026, 1, 4, tzinfo=timezone.utc), ), ] @@ -184,8 +190,7 @@ def test_default_shows_unfinished_and_latest_finished(self): filtered = filter_endpoints_for_listing(endpoints) assert [endpoint.name for endpoint in filtered] == [ - "clauding", - "failed-new", + "prototyping", "running", ] @@ -207,7 +212,7 @@ def test_default_shows_latest_finished_when_none_are_unfinished(self): assert [endpoint.name for endpoint in filtered] == ["failed-new"] - def test_default_shows_latest_finished_and_unfinished_in_watch(self): + def test_default_hides_latest_finished_when_unfinished_exist(self): endpoints = [ _get_endpoint( name="failed-new", @@ -215,15 +220,15 @@ def test_default_shows_latest_finished_and_unfinished_in_watch(self): created_at=datetime(2026, 1, 3, tzinfo=timezone.utc), ), _get_endpoint( - name="clauding", - status=EndpointStatus.CLAUDING, + name="prototyping", + status=EndpointStatus.PROTOTYPING, created_at=datetime(2026, 1, 4, tzinfo=timezone.utc), ), ] filtered = filter_endpoints_for_listing(endpoints) - assert [endpoint.name for endpoint in filtered] == ["clauding", "failed-new"] + assert [endpoint.name for endpoint in filtered] == ["prototyping"] def test_all_shows_all_sorted_newest_first(self): endpoints = [ diff --git a/src/tests/_internal/cli/utils/test_preset.py b/src/tests/_internal/cli/utils/test_preset.py index 2fe75ebc64..361cc33eb5 100644 --- a/src/tests/_internal/cli/utils/test_preset.py +++ b/src/tests/_internal/cli/utils/test_preset.py @@ -1,124 +1,172 @@ from dstack._internal.cli.utils.preset import get_endpoint_presets_table +from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.endpoint_presets import ( EndpointPreset, - EndpointPresetReplicaSpecGroup, + EndpointPresetRecipe, + EndpointPresetValidation, + EndpointPresetValidationReplica, ) class TestGetEndpointPresetsTable: def test_shows_single_group_preset(self): - preset = EndpointPreset( - name="qwen-qwen3-0-6b", + preset = _endpoint_preset( model="Qwen/Qwen3-0.6B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( - { - "name": "0", - "resources": {"gpu": "16GB"}, - "tested_resources": [ - { - "cpu": 9, - "memory": "50GB", - "disk": "60GB", - "gpu": {"name": "A40", "memory": "48GB", "count": 1}, - } - ], - } - ) - ], + recipe_id="vllm-a40", + service={"resources": {"gpu": "nvidia:16GB"}}, ) table = get_endpoint_presets_table([preset]) - assert table.columns[0]._cells == ["qwen-qwen3-0-6b"] - assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B"] - assert table.columns[2]._cells == [ - "cpu=2.. mem=8GB.. disk=100GB.. gpu=16GB:1..", + assert [column.header for column in table.columns] == ["MODEL", "GPU"] + assert table.columns[0]._cells == ["[bold]Qwen/Qwen3-0.6B[/]"] + assert table.columns[1]._cells == [ + "nvidia:16GB:1..", + ] + + def test_verbose_shows_full_resources(self): + preset = _endpoint_preset( + model="Qwen/Qwen3-0.6B", + recipe_id="vllm-a40", + service={"resources": {"gpu": "nvidia:16GB"}}, + ) + + table = get_endpoint_presets_table([preset], verbose=True) + + assert [column.header for column in table.columns] == ["MODEL", "RESOURCES"] + assert table.columns[0]._cells == ["[bold]Qwen/Qwen3-0.6B[/]"] + assert table.columns[1]._cells == [ + "cpu=2.. mem=8GB.. disk=100GB.. gpu=nvidia:16GB:1..", ] def test_shows_single_implicit_group_once_even_with_multiple_tested_replicas(self): - preset = EndpointPreset( - name="qwen", + preset = _endpoint_preset( model="Qwen/Qwen3-0.6B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( - { - "name": "0", - "resources": {"gpu": "16GB"}, - "tested_resources": [ - { - "cpu": 9, - "memory": "50GB", - "disk": "60GB", - "gpu": {"name": "A40", "memory": "48GB", "count": 1}, - }, - { - "cpu": 9, - "memory": "50GB", - "disk": "60GB", - "gpu": {"name": "A40", "memory": "48GB", "count": 1}, - }, - ], - } - ) + recipe_id="vllm-a40", + service={"resources": {"gpu": "nvidia:16GB"}}, + validation_replicas=[ + { + "resources": [ + _exact_resources(), + _exact_resources(), + ] + } ], ) table = get_endpoint_presets_table([preset]) - assert table.columns[0]._cells == ["qwen"] - assert table.columns[1]._cells == ["Qwen/Qwen3-0.6B"] - assert table.columns[2]._cells == [ - "cpu=2.. mem=8GB.. disk=100GB.. gpu=16GB:1..", + assert table.columns[0]._cells == ["[bold]Qwen/Qwen3-0.6B[/]"] + assert table.columns[1]._cells == [ + "nvidia:16GB:1..", ] def test_shows_grouped_replica_specs(self): - preset = EndpointPreset( - name="qwen-pd", + preset = _endpoint_preset( model="Qwen/Qwen3-30B-A3B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( + recipe_id="pd-l4", + service={ + "replicas": [ { "name": "router", + "count": 1, + "commands": ["python router.py"], "resources": {"cpu": 4}, - "tested_resources": [ - {"cpu": 8, "memory": "16GB", "disk": "100GB", "gpu": 0} - ], - } - ), - EndpointPresetReplicaSpecGroup.parse_obj( + }, { "name": "worker", - "resources": {"gpu": "24GB"}, - "tested_resources": [ - { - "cpu": 14, - "memory": "64GB", - "disk": "200GB", - "gpu": {"name": "L4", "memory": "24GB", "count": 1}, - }, - { - "cpu": 14, - "memory": "64GB", - "disk": "200GB", - "gpu": {"name": "L4", "memory": "24GB", "count": 1}, - }, - ], - } - ), + "count": 2, + "commands": ["vllm serve Qwen/Qwen3-30B-A3B"], + "resources": {"gpu": "nvidia:24GB"}, + }, + ] + }, + ) + + table = get_endpoint_presets_table([preset]) + + assert table.columns[0]._cells == [ + "[bold]Qwen/Qwen3-30B-A3B[/]", + "[secondary] group=router[/]", + "[secondary] group=worker[/]", + ] + assert table.columns[1]._cells == [ + "", + "-", + "nvidia:24GB:1..", + ] + + def test_groups_multiple_recipes_by_model(self): + preset = EndpointPreset( + model="Qwen/Qwen3-0.6B", + recipes=[ + _endpoint_preset( + model="Qwen/Qwen3-0.6B", + recipe_id="vllm-a40", + service={"resources": {"gpu": "nvidia:16GB"}}, + ).recipes[0], + _endpoint_preset( + model="Qwen/Qwen3-0.6B", + recipe_id="vllm-l4", + service={"resources": {"gpu": "nvidia:24GB"}}, + ).recipes[0], ], ) table = get_endpoint_presets_table([preset]) assert table.columns[0]._cells == [ - "qwen-pd", - " group=router", - " group=worker", + "[bold]Qwen/Qwen3-0.6B[/]", + "[secondary] recipe=0[/]", + "[secondary] recipe=1[/]", ] - assert table.columns[1]._cells == ["Qwen/Qwen3-30B-A3B", "", ""] - assert table.columns[2]._cells == [ + assert table.columns[1]._cells == [ "", - "cpu=4 mem=8GB.. disk=100GB..", - "cpu=2.. mem=8GB.. disk=100GB.. gpu=24GB:1..", + "nvidia:16GB:1..", + "nvidia:24GB:1..", ] + + +def _endpoint_preset( + *, + model: str, + recipe_id: str, + service: dict, + validation_replicas: list[dict] | None = None, +) -> EndpointPreset: + service_data = { + "type": "service", + "port": 8000, + "model": model, + **service, + } + if "replicas" not in service_data: + service_data["commands"] = [f"vllm serve {model}"] + return EndpointPreset( + model=model, + recipes=[ + EndpointPresetRecipe( + id=recipe_id, + service=ServiceConfiguration.parse_obj(service_data), + validations=[ + EndpointPresetValidation( + replicas=[ + EndpointPresetValidationReplica.parse_obj(replica) + for replica in ( + validation_replicas or [{"resources": [_exact_resources()]}] + ) + ] + ) + ], + ) + ], + ) + + +def _exact_resources() -> dict: + return { + "cpu": 9, + "memory": "50GB", + "disk": "60GB", + "gpu": {"name": "A40", "memory": "48GB", "count": 1}, + } diff --git a/src/tests/_internal/core/models/test_endpoints.py b/src/tests/_internal/core/models/test_endpoints.py index 1656a81716..ba562a8aa7 100644 --- a/src/tests/_internal/core/models/test_endpoints.py +++ b/src/tests/_internal/core/models/test_endpoints.py @@ -17,7 +17,6 @@ def test_parses_endpoint_configuration(self): "fleets": ["gpu-fleet"], "creation_policy": "reuse", "preset_policy": "reuse", - "max_agent_budget": 2.0, } ) @@ -31,24 +30,11 @@ def test_parses_endpoint_configuration(self): assert fleet.name == "gpu-fleet" assert conf.creation_policy == CreationPolicy.REUSE assert conf.preset_policy == EndpointPresetPolicy.REUSE - assert conf.max_agent_budget == 2.0 def test_defaults_to_reuse_or_create_preset_policy(self): conf = EndpointConfiguration(name="qwen-endpoint", model="Qwen/Qwen3-0.6B") assert conf.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE - assert conf.max_agent_budget is None - - def test_rejects_non_positive_agent_budget(self): - with pytest.raises(ConfigurationError): - parse_apply_configuration( - { - "type": "endpoint", - "name": "qwen-endpoint", - "model": "Qwen/Qwen3-0.6B", - "max_agent_budget": 0, - } - ) def test_rejects_unknown_fields(self): with pytest.raises(ConfigurationError): diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index 68bc3b679d..253cf592ab 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -10,7 +10,11 @@ from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import ServiceConfiguration -from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.endpoints import ( + EndpointConfiguration, + EndpointPresetPolicy, + EndpointStatus, +) from dstack._internal.core.models.envs import Env from dstack._internal.core.models.instances import ( Disk, @@ -36,6 +40,7 @@ from dstack._internal.server.services.endpoints.planning import EndpointPresetPlanningResult from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( + create_fleet, create_job, create_project, create_repo, @@ -57,6 +62,7 @@ def __init__(self, result: AgentProvisioningResult | None = None, side_effect=No return_value=result or AgentProvisioningResult(), side_effect=side_effect, ) + self.abort_endpoint = AsyncMock(return_value=True) def is_enabled(self) -> bool: return True @@ -247,18 +253,18 @@ def _get_verified_agent_result( run_id=run.id, run_name=run.run_name, service_yaml=f"type: service\nname: {run.run_name}\n", - verification_summary="Agent verified the model endpoint.", ), ) -def _get_agent_run_name(suffix: str = "candidate") -> str: - return f"agent-{suffix}" +def _get_agent_run_name(suffix: str = "1") -> str: + return f"qwen-endpoint-{suffix}" def _make_preset_plan(run_name: str = "qwen-endpoint-serving") -> Mock: preset_plan = Mock() - preset_plan.preset.name = "qwen" + preset_plan.preset.model = "Qwen/Qwen3-0.6B" + preset_plan.recipe.id = "vllm-t4" preset_plan.run_plan.run_spec = RunSpec( run_name=run_name, configuration=ServiceConfiguration.parse_obj( @@ -398,6 +404,7 @@ async def test_fails_when_preset_run_name_exists_without_link( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) run = await _create_backing_service_run( session=session, endpoint_model=endpoint_model, @@ -442,6 +449,7 @@ async def test_fails_when_preset_run_name_is_taken_by_another_user( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) repo = await create_repo(session=session, project_id=endpoint_model.project_id) another_user = await create_user(session=session, name="another-user") run = await create_run( @@ -490,6 +498,7 @@ async def test_fails_when_preset_run_name_is_taken_by_pre_existing_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) repo = await create_repo(session=session, project_id=endpoint_model.project_id) run = await create_run( session=session, @@ -521,10 +530,11 @@ async def test_fails_when_preset_run_name_is_taken_by_pre_existing_run( assert run.deleted is False find_preset_planning_result_mock.assert_awaited_once() - async def test_preset_run_name_conflict_does_not_block_clauding_without_preset( + async def test_preset_run_name_conflict_does_not_block_prototyping_without_preset( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) run = await _create_backing_service_run( session=session, endpoint_model=endpoint_model, @@ -548,7 +558,7 @@ async def test_preset_run_name_conflict_does_not_block_clauding_without_preset( await session.refresh(endpoint_model) await session.refresh(run) - assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status == EndpointStatus.PROTOTYPING assert endpoint_model.status_message is None assert endpoint_model.service_run_id is None assert endpoint_model.provisioning_method == "agent" @@ -564,30 +574,14 @@ async def test_finished_same_name_run_does_not_block_preset_submission( session=session, user_ssh_public_key="ssh-rsa test", ) + await create_fleet(session=session, project=endpoint_model.project) run = await _create_backing_service_run( session=session, endpoint_model=endpoint_model, status=RunStatus.DONE, link_endpoint=False, ) - preset_plan = Mock() - preset_plan.preset.name = "qwen" - preset_plan.run_plan.run_spec = RunSpec( - run_name="qwen-endpoint-serving", - configuration=ServiceConfiguration.parse_obj( - { - "type": "service", - "name": "qwen-endpoint-serving", - "commands": [ - "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", - ], - "port": 8000, - "model": "Qwen/Qwen3-0.6B", - "resources": {"gpu": "16GB"}, - } - ), - ) - preset_plan.run_plan.current_resource = None + preset_plan = _make_preset_plan() with ( patch( @@ -608,7 +602,7 @@ async def test_finished_same_name_run_does_not_block_preset_submission( await session.refresh(run) assert endpoint_model.status == EndpointStatus.PROVISIONING assert endpoint_model.status_message is None - assert endpoint_model.provisioning_method == "preset:qwen" + assert endpoint_model.provisioning_method == "preset:Qwen/Qwen3-0.6B#vllm-t4" assert run.status == RunStatus.DONE assert run.deleted is False find_preset_planning_result_mock.assert_awaited_once() @@ -618,6 +612,7 @@ async def test_submitted_to_failed_without_provisioning_path( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) with patch( "dstack._internal.server.background.pipeline_tasks.endpoints." @@ -642,10 +637,11 @@ async def test_submitted_to_failed_without_provisioning_path( "DSTACK_AGENT_ANTHROPIC_API_KEY is not set.)" ) - async def test_submitted_to_clauding_with_agent( + async def test_submitted_to_prototyping_with_agent( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) agent_service = _FakeAgentService() with ( @@ -662,14 +658,66 @@ async def test_submitted_to_clauding_with_agent( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status == EndpointStatus.PROTOTYPING assert endpoint_model.status_message is None assert endpoint_model.service_run_id is None assert endpoint_model.provisioning_method == "agent" agent_service.provision_endpoint.assert_not_awaited() events = await list_events(session) assert len(events) == 1 - assert events[0].message == "Endpoint status changed SUBMITTED -> CLAUDING" + assert events[0].message == "Endpoint status changed SUBMITTED -> PROTOTYPING" + + async def test_submitted_to_failed_without_fleets_even_when_agent_enabled( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + agent_service = _FakeAgentService() + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "The project has no fleets. Create one before submitting an endpoint." + ) + assert endpoint_model.service_run_id is None + assert endpoint_model.provisioning_method is None + agent_service.provision_endpoint.assert_not_awaited() + + async def test_submitted_reuse_to_failed_without_fleets( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model(session=session) + configuration = EndpointConfiguration.__response__.parse_raw(endpoint_model.configuration) + configuration.preset_policy = EndpointPresetPolicy.REUSE + endpoint_model.configuration = configuration.json() + await session.commit() + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "The project has no fleets. Create one before submitting an endpoint." + ) + assert endpoint_model.service_run_id is None + assert endpoint_model.provisioning_method is None async def test_project_user_cannot_start_agent( self, test_db, session: AsyncSession, worker: EndpointWorker @@ -679,6 +727,7 @@ async def test_project_user_cannot_start_agent( user_global_role=GlobalRole.USER, project_role=ProjectRole.USER, ) + await create_fleet(session=session, project=endpoint_model.project) agent_service = _FakeAgentService() with ( @@ -711,6 +760,7 @@ async def test_submitted_to_provisioning_with_matching_preset( session=session, user_ssh_public_key="ssh-rsa test", ) + await create_fleet(session=session, project=endpoint_model.project) repo = await create_repo(session=session, project_id=endpoint_model.project_id) run = await create_run( session=session, @@ -719,24 +769,7 @@ async def test_submitted_to_provisioning_with_matching_preset( user=endpoint_model.user, run_name="qwen-endpoint-submitted", ) - preset_plan = Mock() - preset_plan.preset.name = "qwen" - preset_plan.run_plan.run_spec = RunSpec( - run_name="qwen-endpoint-serving", - configuration=ServiceConfiguration.parse_obj( - { - "type": "service", - "name": "qwen-endpoint-serving", - "commands": [ - "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", - ], - "port": 8000, - "model": "Qwen/Qwen3-0.6B", - "resources": {"gpu": "16GB"}, - } - ), - ) - preset_plan.run_plan.current_resource = None + preset_plan = _make_preset_plan() with ( patch( @@ -757,7 +790,7 @@ async def test_submitted_to_provisioning_with_matching_preset( assert endpoint_model.status == EndpointStatus.PROVISIONING assert endpoint_model.status_message is None assert endpoint_model.service_run_id == run.id - assert endpoint_model.provisioning_method == "preset:qwen" + assert endpoint_model.provisioning_method == "preset:Qwen/Qwen3-0.6B#vllm-t4" submissions = await _get_endpoint_run_submissions( session=session, endpoint_model=endpoint_model, @@ -778,24 +811,8 @@ async def test_submitted_to_failed_when_preset_submission_fails( session=session, user_ssh_public_key="ssh-rsa test", ) - preset_plan = Mock() - preset_plan.preset.name = "qwen" - preset_plan.run_plan.run_spec = RunSpec( - run_name="qwen-endpoint-serving", - configuration=ServiceConfiguration.parse_obj( - { - "type": "service", - "name": "qwen-endpoint-serving", - "commands": [ - "vllm serve Qwen/Qwen3-0.6B --host 0.0.0.0 --port 8000", - ], - "port": 8000, - "model": "Qwen/Qwen3-0.6B", - "resources": {"gpu": "16GB"}, - } - ), - ) - preset_plan.run_plan.current_resource = None + await create_fleet(session=session, project=endpoint_model.project) + preset_plan = _make_preset_plan() with ( patch( @@ -827,12 +844,12 @@ async def test_submitted_to_failed_when_preset_submission_fails( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) class TestEndpointWorkerProvisioning: - async def test_clauding_does_not_fail_on_legacy_same_name_run_conflict( + async def test_prototyping_does_not_fail_on_legacy_same_name_run_conflict( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) legacy_run = await _create_backing_service_run( session=session, @@ -876,14 +893,14 @@ async def test_clauding_does_not_fail_on_legacy_same_name_run_conflict( assert agent_run.deleted is False events = await list_events(session) assert len(events) == 1 - assert events[0].message == "Endpoint status changed CLAUDING -> RUNNING" + assert events[0].message == "Endpoint status changed PROTOTYPING -> RUNNING" async def test_fails_when_agent_reports_foreign_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) repo = await create_repo(session=session, project_id=endpoint_model.project_id) another_user = await create_user(session=session, name="another-user") @@ -907,7 +924,7 @@ async def test_fails_when_agent_reports_foreign_run( await session.refresh(run) assert endpoint_model.status == EndpointStatus.FAILED assert endpoint_model.status_message == ( - "Run 'agent-candidate' is not owned by the endpoint user" + "Run 'qwen-endpoint-1' is not owned by the endpoint user" ) assert endpoint_model.service_run_id is None assert run.status == RunStatus.RUNNING @@ -915,14 +932,45 @@ async def test_fails_when_agent_reports_foreign_run( events = await list_events(session) assert len(events) == 1 assert ( - events[0].message == "Endpoint status changed CLAUDING -> FAILED " - "(Run 'agent-candidate' is not owned by the endpoint user)" + events[0].message == "Endpoint status changed PROTOTYPING -> FAILED " + "(Run 'qwen-endpoint-1' is not owned by the endpoint user)" + ) + + async def test_fails_when_agent_final_run_name_is_not_submission_number( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROTOTYPING, + ) + run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name="agent-run", + ) + await session.commit() + agent_service = _FakeAgentService(result=_get_verified_agent_result(run)) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.status_message == ( + "Server agent final service run name must be 'qwen-endpoint-'" ) + assert endpoint_model.service_run_id is None + assert run.status == RunStatus.RUNNING async def test_agent_creates_ready_service_and_endpoint_becomes_running( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model(session=session) + await create_fleet(session=session, project=endpoint_model.project) preset_service = Mock() preset_service.save_preset = AsyncMock( side_effect=lambda project_name, preset, comments: preset @@ -951,7 +999,7 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): ): await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status == EndpointStatus.PROTOTYPING await _lock_endpoint_model(session=session, endpoint_model=endpoint_model) await worker.process(_endpoint_to_pipeline_item(endpoint_model)) @@ -975,7 +1023,7 @@ async def test_agent_success_after_stop_keeps_endpoint_stopping_and_links_run( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1045,7 +1093,7 @@ async def test_agent_reported_run_id_links_backing_service_run( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1072,7 +1120,6 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): run_id=run.id, run_name=run.run_name, service_yaml=f"type: service\nname: {run.run_name}\n", - verification_summary="Agent verified the model endpoint.", ), ) @@ -1106,12 +1153,80 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): assert submissions[0].run_id == endpoint_model.service_run_id preset_service.save_preset.assert_awaited_once() - async def test_clauding_linked_run_waits_without_showing_provisioning( + async def test_agent_success_stops_non_final_submitted_runs( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROTOTYPING, + ) + endpoint_model.provisioning_method = "agent" + probe_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=f"{endpoint_model.name}-1", + ) + final_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=f"{endpoint_model.name}-2", + ) + endpoint_model.service_run_id = None + await session.commit() + agent_service = _FakeAgentService( + result=AgentProvisioningResult( + run_id=final_run.id, + run_name=final_run.run_name, + submitted_run_ids=(probe_run.id, final_run.id), + final_report=AgentFinalReport( + success=True, + run_id=final_run.id, + run_name=final_run.run_name, + service_yaml=f"type: service\nname: {final_run.run_name}\n", + ), + ) + ) + preset_service = Mock() + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + await session.refresh(probe_run) + await session.refresh(final_run) + assert endpoint_model.status == EndpointStatus.RUNNING + assert endpoint_model.service_run_id == final_run.id + assert probe_run.status == RunStatus.TERMINATING + assert final_run.status == RunStatus.RUNNING + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert [submission.run_id for submission in submissions] == [probe_run.id, final_run.id] + assert [submission.submission_num for submission in submissions] == [1, 2] + preset_service.save_preset.assert_awaited_once() + + async def test_prototyping_linked_run_waits_without_showing_provisioning( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await _create_backing_service_run( @@ -1123,7 +1238,7 @@ async def test_clauding_linked_run_waits_without_showing_provisioning( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status == EndpointStatus.PROTOTYPING assert endpoint_model.status_message is None events = await list_events(session) assert events == [] @@ -1133,7 +1248,7 @@ async def test_agent_reported_run_without_verification_report_fails_endpoint( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1165,14 +1280,14 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): await session.refresh(endpoint_model) assert endpoint_model.status == EndpointStatus.FAILED assert endpoint_model.service_run_id is None - assert endpoint_model.status_message == "Server agent did not return a verification report" + assert endpoint_model.status_message == "Server agent did not return a final report" async def test_agent_failure_fails_endpoint( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1191,12 +1306,12 @@ async def test_agent_failure_fails_endpoint( assert endpoint_model.status_message == "agent could not find a deployable recipe" agent_service.provision_endpoint.assert_awaited_once() - async def test_agent_in_progress_keeps_endpoint_clauding( + async def test_agent_in_progress_keeps_endpoint_prototyping( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1209,20 +1324,20 @@ async def test_agent_in_progress_keeps_endpoint_clauding( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status == EndpointStatus.PROTOTYPING assert endpoint_model.status_message is None assert endpoint_model.service_run_id is None agent_service.provision_endpoint.assert_awaited_once() - async def test_agent_in_progress_records_candidate_run_submission( + async def test_agent_in_progress_records_submitted_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" - candidate_run = await _create_backing_service_run( + submitted_run = await _create_backing_service_run( session=session, endpoint_model=endpoint_model, link_endpoint=False, @@ -1233,7 +1348,46 @@ async def test_agent_in_progress_records_candidate_run_submission( agent_service = _FakeAgentService( result=AgentProvisioningResult( in_progress=True, - candidate_run_ids=(candidate_run.id,), + submitted_run_ids=(submitted_run.id,), + ) + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.PROTOTYPING + assert endpoint_model.service_run_id is None + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert len(submissions) == 1 + assert submissions[0].run_id == submitted_run.id + + async def test_agent_in_progress_records_submitted_run_by_name( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROTOTYPING, + ) + endpoint_model.provisioning_method = "agent" + submitted_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name=f"{endpoint_model.name}-1", + ) + endpoint_model.service_run_id = None + await session.commit() + agent_service = _FakeAgentService( + result=AgentProvisioningResult( + in_progress=True, + submitted_run_names=(submitted_run.run_name,), ) ) @@ -1244,21 +1398,56 @@ async def test_agent_in_progress_records_candidate_run_submission( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - assert endpoint_model.status == EndpointStatus.CLAUDING + assert endpoint_model.status == EndpointStatus.PROTOTYPING assert endpoint_model.service_run_id is None submissions = await _get_endpoint_run_submissions( session=session, endpoint_model=endpoint_model, ) assert len(submissions) == 1 - assert submissions[0].run_id == candidate_run.id + assert submissions[0].run_id == submitted_run.id + + async def test_agent_in_progress_ignores_non_strict_submitted_run_name( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROTOTYPING, + ) + endpoint_model.provisioning_method = "agent" + submitted_run = await _create_backing_service_run( + session=session, + endpoint_model=endpoint_model, + link_endpoint=False, + run_name="agent-run", + ) + endpoint_model.service_run_id = None + await session.commit() + agent_service = _FakeAgentService( + result=AgentProvisioningResult( + in_progress=True, + submitted_run_names=(submitted_run.run_name,), + ) + ) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + submissions = await _get_endpoint_run_submissions( + session=session, + endpoint_model=endpoint_model, + ) + assert submissions == [] async def test_agent_error_status_message_is_compact( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1286,7 +1475,7 @@ async def test_agent_failure_report_status_message_is_compact( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1321,7 +1510,7 @@ async def test_agent_failure_does_not_stop_unlinked_same_name_run( ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" await session.commit() @@ -1363,15 +1552,15 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): assert run.status == RunStatus.RUNNING agent_service.provision_endpoint.assert_awaited_once() - async def test_agent_failure_stops_recorded_candidate_run( + async def test_agent_failure_stops_recorded_submitted_run( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( session=session, - status=EndpointStatus.CLAUDING, + status=EndpointStatus.PROTOTYPING, ) endpoint_model.provisioning_method = "agent" - candidate_run = await _create_backing_service_run( + submitted_run = await _create_backing_service_run( session=session, endpoint_model=endpoint_model, link_endpoint=False, @@ -1382,7 +1571,7 @@ async def test_agent_failure_stops_recorded_candidate_run( agent_service = _FakeAgentService( result=AgentProvisioningResult( error="agent could not verify the service", - candidate_run_ids=(candidate_run.id,), + submitted_run_ids=(submitted_run.id,), ) ) @@ -1393,17 +1582,17 @@ async def test_agent_failure_stops_recorded_candidate_run( await worker.process(_endpoint_to_pipeline_item(endpoint_model)) await session.refresh(endpoint_model) - await session.refresh(candidate_run) + await session.refresh(submitted_run) assert endpoint_model.status == EndpointStatus.FAILED assert endpoint_model.status_message == "agent could not verify the service" assert endpoint_model.service_run_id is None - assert candidate_run.status == RunStatus.TERMINATING + assert submitted_run.status == RunStatus.TERMINATING submissions = await _get_endpoint_run_submissions( session=session, endpoint_model=endpoint_model, ) assert len(submissions) == 1 - assert submissions[0].run_id == candidate_run.id + assert submissions[0].run_id == submitted_run.id async def test_waits_when_backing_run_is_not_ready( self, test_db, session: AsyncSession, worker: EndpointWorker @@ -1471,7 +1660,9 @@ async def test_saves_preset_when_agent_backing_service_becomes_running( assert preset_service.save_preset.await_args.args[0] == "test_project" saved_preset = preset_service.save_preset.await_args.args[1] assert saved_preset.model == "Qwen/Qwen3-0.6B" - assert [group.name for group in saved_preset.replica_spec_groups] == ["0"] + assert len(saved_preset.recipes) == 1 + assert saved_preset.recipes[0].service.resources.gpu is not None + assert saved_preset.recipes[0].validations[0].replicas[0].resources[0].gpu is not None comments = preset_service.save_preset.await_args.kwargs["comments"] assert f"endpoint: {endpoint_model.name}" in comments @@ -1698,6 +1889,52 @@ async def test_marks_endpoint_stopped_without_backing_run( events = await list_events(session) assert events[0].message == "Endpoint status changed STOPPING -> STOPPED" + async def test_aborts_agent_before_marking_endpoint_stopped( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.STOPPING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + abort_agent_endpoint = AsyncMock(return_value=True) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.abort_agent_endpoint", + abort_agent_endpoint, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + abort_agent_endpoint.assert_awaited_once() + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.STOPPED + events = await list_events(session) + assert events[0].message == "Endpoint status changed STOPPING -> STOPPED" + + async def test_waits_when_agent_abort_is_still_pending( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.STOPPING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + abort_agent_endpoint = AsyncMock(return_value=False) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.abort_agent_endpoint", + abort_agent_endpoint, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + abort_agent_endpoint.assert_awaited_once() + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.STOPPING + events = await list_events(session) + assert events == [] + async def test_stops_backing_run_before_stopping_endpoint( self, test_db, session: AsyncSession, worker: EndpointWorker ): @@ -1719,7 +1956,7 @@ async def test_stops_backing_run_before_stopping_endpoint( assert run.status == RunStatus.TERMINATING assert run.deleted is False - async def test_stops_recorded_candidate_run_before_stopping_endpoint( + async def test_stops_recorded_submitted_run_before_stopping_endpoint( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( @@ -1748,7 +1985,7 @@ async def test_stops_recorded_candidate_run_before_stopping_endpoint( assert endpoint_model.service_run_id is None assert run.status == RunStatus.TERMINATING - async def test_waits_for_terminating_recorded_candidate_run_before_stopping_endpoint( + async def test_waits_for_terminating_recorded_submitted_run_before_stopping_endpoint( self, test_db, session: AsyncSession, worker: EndpointWorker ): endpoint_model = await _create_endpoint_model( diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index 918e042e20..20e12cd351 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -10,14 +10,16 @@ from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import ServiceConfiguration -from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointStatus +from dstack._internal.core.models.endpoints import ( + EndpointConfiguration, + EndpointStatus, +) from dstack._internal.core.models.instances import ( InstanceAvailability, InstanceOfferWithAvailability, ) from dstack._internal.core.models.runs import RunSpec, RunStatus from dstack._internal.core.models.users import GlobalRole, ProjectRole -from dstack._internal.server import settings from dstack._internal.server.models import EndpointModel from dstack._internal.server.services.endpoints.agent import AgentPlan from dstack._internal.server.services.endpoints.planning import ( @@ -26,10 +28,13 @@ ) from dstack._internal.server.services.endpoints.presets import ( EndpointPreset, - EndpointPresetReplicaSpecGroup, + EndpointPresetRecipe, + EndpointPresetValidation, + EndpointPresetValidationReplica, ) from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( + create_fleet, create_project, create_repo, create_run, @@ -55,6 +60,7 @@ async def test_returns_none_provisioning_plan( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) + await create_fleet(session=session, project=project) with patch( "dstack._internal.server.services.endpoints.find_preset_planning_result", @@ -133,14 +139,14 @@ async def test_returns_create_plan_when_existing_endpoint_is_terminal( @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_returns_agent_provisioning_plan( - self, test_db, session: AsyncSession, client: AsyncClient, monkeypatch: pytest.MonkeyPatch + self, test_db, session: AsyncSession, client: AsyncClient ): - monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MAX_BUDGET", 4.0) user = await create_user(session, global_role=GlobalRole.USER) project = await create_project(session) await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) + await create_fleet(session=session, project=project) agent_service = Mock() agent_service.is_enabled.return_value = True agent_service.get_plan.return_value = AgentPlan(model="test-agent") @@ -163,7 +169,6 @@ async def test_returns_agent_provisioning_plan( "type": "endpoint", "name": "qwen-endpoint", "model": "Qwen/Qwen3-0.6B", - "max_agent_budget": 2.0, }, "configuration_path": "endpoint.dstack.yml", }, @@ -175,10 +180,131 @@ async def test_returns_agent_provisioning_plan( assert body["provisioning_plan"] == { "type": "agent", "agent_model": "test-agent", - "max_budget": 2.0, "reason": None, } + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_no_agent_plan_without_existing_fleets( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + agent_service = Mock() + agent_service.is_enabled.return_value = True + agent_service.get_plan.return_value = AgentPlan(model="test-agent") + + with ( + patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.services.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + }, + "configuration_path": "endpoint.dstack.yml", + }, + ) + + assert response.status_code == 200, response.json() + assert response.json()["provisioning_plan"] == { + "type": "none", + "reason": "The project has no fleets. Create one before submitting an endpoint.", + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_no_fleets_plan_for_reuse_without_existing_fleets( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "preset_policy": "reuse", + }, + "configuration_path": "endpoint.dstack.yml", + }, + ) + + assert response.status_code == 200, response.json() + assert response.json()["provisioning_plan"] == { + "type": "none", + "reason": "The project has no fleets. Create one before submitting an endpoint.", + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_no_agent_plan_when_configured_fleet_does_not_match( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + await create_fleet(session=session, project=project, name="existing-fleet") + agent_service = Mock() + agent_service.is_enabled.return_value = True + agent_service.get_plan.return_value = AgentPlan(model="test-agent") + + with ( + patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ), + patch( + "dstack._internal.server.services.endpoints.get_agent_service", + return_value=agent_service, + ), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/get_plan", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "fleets": ["missing-fleet"], + }, + "configuration_path": "endpoint.dstack.yml", + }, + ) + + assert response.status_code == 200, response.json() + assert response.json()["provisioning_plan"] == { + "type": "none", + "reason": ( + "No fleets match the endpoint configuration. Create a fleet or update `fleets` " + "before submitting an endpoint." + ), + } + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_returns_no_agent_plan_for_project_user( @@ -189,6 +315,7 @@ async def test_returns_no_agent_plan_for_project_user( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) + await create_fleet(session=session, project=project) agent_service = Mock() agent_service.is_enabled.return_value = True @@ -234,6 +361,7 @@ async def test_returns_preset_provisioning_plan( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) + await create_fleet(session=session, project=project) with patch( "dstack._internal.server.services.endpoints.find_preset_planning_result", @@ -257,9 +385,9 @@ async def test_returns_preset_provisioning_plan( body = response.json() assert body["provisioning_plan"]["type"] == "preset" assert body["preset_policy"] == "reuse-or-create" - assert body["provisioning_plan"]["preset_name"] == "qwen" + assert body["provisioning_plan"]["preset_model"] == "Qwen/Qwen3-0.6B" + assert body["provisioning_plan"]["recipe_id"] == "vllm-t4" assert body["provisioning_plan"]["service_name"] == "qwen-endpoint-serving" - assert body["provisioning_plan"]["replica_spec_groups"][0]["name"] == "0" assert body["provisioning_plan"]["job_offers"][0]["replica_group"] == "0" assert body["provisioning_plan"]["job_offers"][0]["resources"]["gpu"] is not None assert body["provisioning_plan"]["job_offers"][0]["spot"] is None @@ -283,6 +411,7 @@ async def test_creates_endpoint(self, test_db, session: AsyncSession, client: As await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) + await create_fleet(session=session, project=project) response = await client.post( f"/api/project/{project.name}/endpoints/create", @@ -319,6 +448,67 @@ async def test_creates_endpoint(self, test_db, session: AsyncSession, client: As assert len(events) == 1 assert events[0].message == "Endpoint created. Status: SUBMITTED" + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_rejects_create_without_existing_fleets( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + + with patch( + "dstack._internal.server.services.endpoints.find_preset_planning_result", + new=AsyncMock(return_value=EndpointPresetPlanningResult()), + ): + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"][0]["msg"] == ( + "The project has no fleets. Create one before submitting an endpoint." + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_rejects_reuse_without_existing_fleets( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + "preset_policy": "reuse", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"][0]["msg"] == ( + "The project has no fleets. Create one before submitting an endpoint." + ) + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_resubmits_terminal_endpoint( @@ -329,6 +519,7 @@ async def test_resubmits_terminal_endpoint( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.ADMIN ) + await create_fleet(session=session, project=project) previous_endpoint = await _create_endpoint_model( session=session, project=project, @@ -361,6 +552,45 @@ async def test_resubmits_terminal_endpoint( res = await session.execute(select(EndpointModel)) assert len(res.scalars().all()) == 1 + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_rejects_resubmit_terminal_endpoint_without_existing_fleets( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + previous_endpoint = await _create_endpoint_model( + session=session, + project=project, + user=user, + status=EndpointStatus.FAILED, + ) + previous_endpoint.status_message = "old failure" + await session.commit() + + response = await client.post( + f"/api/project/{project.name}/endpoints/create", + headers=get_auth_headers(user.token), + json={ + "configuration": { + "type": "endpoint", + "name": "qwen-endpoint", + "model": "Qwen/Qwen3-0.6B", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"][0]["msg"] == ( + "The project has no fleets. Create one before submitting an endpoint." + ) + await session.refresh(previous_endpoint) + assert previous_endpoint.status == EndpointStatus.FAILED + assert previous_endpoint.status_message == "old failure" + @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) async def test_rejects_project_user_when_create_requires_agent( @@ -371,6 +601,7 @@ async def test_rejects_project_user_when_create_requires_agent( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) + await create_fleet(session=session, project=project) with patch( "dstack._internal.server.services.endpoints.find_preset_planning_result", @@ -403,6 +634,7 @@ async def test_allows_project_user_when_matching_preset_exists( await add_project_member( session=session, project=project, user=user, project_role=ProjectRole.USER ) + await create_fleet(session=session, project=project) with patch( "dstack._internal.server.services.endpoints.find_preset_planning_result", @@ -643,11 +875,10 @@ async def test_lists_endpoint_presets( assert response.status_code == 200, response.json() body = response.json() assert len(body) == 1 - assert body[0]["name"] == "qwen" assert body[0]["model"] == "Qwen/Qwen3-0.6B" - assert body[0]["replica_spec_groups"][0]["name"] == "0" - assert body[0]["replica_spec_groups"][0]["resources"]["gpu"] is not None - assert body[0]["replica_spec_groups"][0]["tested_resources"][0]["gpu"] is not None + assert body[0]["recipes"][0]["id"] == "vllm-t4" + assert body[0]["recipes"][0]["service"]["resources"]["gpu"] is not None + assert body[0]["recipes"][0]["validations"][0]["replicas"][0]["resources"][0]["gpu"] assert preset_service.list_project_names == [project.name] @pytest.mark.asyncio @@ -673,11 +904,11 @@ async def test_deletes_endpoint_preset( response = await client.post( f"/api/project/{project.name}/endpoints/presets/delete", headers=get_auth_headers(user.token), - json={"names": ["qwen"]}, + json={"models": ["Qwen/Qwen3-0.6B"]}, ) assert response.status_code == 200, response.json() - assert preset_service.deleted_names == ["qwen"] + assert preset_service.deleted_models == ["Qwen/Qwen3-0.6B"] assert preset_service.delete_project_names == [project.name] @pytest.mark.asyncio @@ -703,15 +934,14 @@ async def test_gets_endpoint_preset( response = await client.post( f"/api/project/{project.name}/endpoints/presets/get", headers=get_auth_headers(user.token), - json={"name": "qwen"}, + json={"model": "Qwen/Qwen3-0.6B"}, ) assert response.status_code == 200, response.json() body = response.json() - assert body["name"] == "qwen" assert body["model"] == "Qwen/Qwen3-0.6B" - assert body["service"]["type"] == "service" - assert body["service"]["name"] == "qwen-endpoint-serving" + assert body["recipes"][0]["service"]["type"] == "service" + assert body["recipes"][0]["service"]["resources"]["gpu"] is not None assert preset_service.get_project_names == [project.name] @@ -721,24 +951,24 @@ def __init__(self, presets): self.list_project_names = [] self.get_project_names = [] self.delete_project_names = [] - self.deleted_names = [] + self.deleted_models = [] async def list_presets(self, project_name): self.list_project_names.append(project_name) return self._presets - async def get_preset(self, project_name, name): + async def get_preset(self, project_name, model): self.get_project_names.append(project_name) for preset in self._presets: - if preset.name == name: + if preset.model == model: return preset return None - async def delete_preset(self, project_name, name): - if name not in {preset.name for preset in self._presets}: - raise FileNotFoundError(name) + async def delete_preset(self, project_name, model): + if model not in {preset.model for preset in self._presets}: + raise FileNotFoundError(model) self.delete_project_names.append(project_name) - self.deleted_names.append(name) + self.deleted_models.append(model) async def _create_endpoint_model( @@ -776,27 +1006,29 @@ def _endpoint_preset_plan() -> EndpointPresetPlan: "resources": {"gpu": "16GB"}, } ) - preset = EndpointPreset( - name="qwen", - model="Qwen/Qwen3-0.6B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj( - { - "name": "0", - "resources": {"gpu": "16GB"}, - "tested_resources": [ + recipe = EndpointPresetRecipe( + id="vllm-t4", + service=service_configuration, + validations=[ + EndpointPresetValidation( + replicas=[ + EndpointPresetValidationReplica.parse_obj( { - "cpu": 4, - "memory": "16GB", - "disk": "100GB", - "gpu": {"name": "T4", "memory": "16GB", "count": 1}, + "resources": [ + { + "cpu": 4, + "memory": "16GB", + "disk": "100GB", + "gpu": {"name": "T4", "memory": "16GB", "count": 1}, + } + ] } - ], - } + ) + ] ) ], - configuration=service_configuration, ) + preset = EndpointPreset(model="Qwen/Qwen3-0.6B", recipes=[recipe]) run_plan = Mock() run_plan.get_effective_run_spec.return_value = RunSpec( run_name="qwen-endpoint-serving", @@ -811,7 +1043,7 @@ def _endpoint_preset_plan() -> EndpointPresetPlan: job_plan.total_offers = 2 job_plan.max_price = 1.25 run_plan.job_plans = [job_plan] - return EndpointPresetPlan(preset=preset, run_plan=run_plan) + return EndpointPresetPlan(preset=preset, recipe=recipe, run_plan=run_plan) def _instance_offer() -> InstanceOfferWithAvailability: diff --git a/src/tests/_internal/server/services/endpoints/test_agent_report.py b/src/tests/_internal/server/services/endpoints/test_agent_report.py index 367c730d55..99192254ba 100644 --- a/src/tests/_internal/server/services/endpoints/test_agent_report.py +++ b/src/tests/_internal/server/services/endpoints/test_agent_report.py @@ -14,25 +14,13 @@ def test_accepts_success_report(self): { "success": True, "run_id": str(run_id), + "run_name": "qwen-serving", "service_yaml": "type: service\nname: qwen-serving\n", - "recipe_sources": ["https://example.com/recipe"], - "verification_summary": "Chat completion request returned 200.", } ) assert report.run_id == run_id - assert report.recipe_sources == ["https://example.com/recipe"] - - def test_rejects_success_without_verification_summary(self): - with pytest.raises(ValidationError, match="verification_summary"): - AgentFinalReport.parse_obj( - { - "success": True, - "run_id": str(uuid.uuid4()), - "run_name": "qwen-serving", - "service_yaml": "type: service\nname: qwen-serving\n", - } - ) + assert report.run_name == "qwen-serving" def test_rejects_success_without_run_id(self): with pytest.raises(ValidationError, match="run_id"): @@ -41,7 +29,6 @@ def test_rejects_success_without_run_id(self): "success": True, "run_name": "qwen-serving", "service_yaml": "type: service\nname: qwen-serving\n", - "verification_summary": "Chat completion request returned 200.", } ) diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index dad7451562..8b82eb22de 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -1,7 +1,11 @@ import json +import signal +import stat +import subprocess import uuid from asyncio import StreamReader from datetime import datetime, timezone +from pathlib import Path import pytest import yaml @@ -15,8 +19,8 @@ from dstack._internal.core.models.profiles import SpotPolicy from dstack._internal.server import settings from dstack._internal.server.models import ( - EndpointAgentAttemptModel, - EndpointAgentAttemptStatus, + EndpointAgentSessionModel, + EndpointAgentSessionStatus, EndpointModel, ) from dstack._internal.server.services.endpoints.agent.claude import ( @@ -24,30 +28,36 @@ _AgentRunnerResult, _AgentWorkspace, _build_claude_command, + _load_submissions, _read_agent_stdout, _run_agent_in_subprocess, _write_trace_record, get_claude_agent_unavailable_reason, ) from dstack._internal.server.services.endpoints.agent.report import AgentFinalReport -from dstack._internal.server.testing.common import create_project, create_user +from dstack._internal.server.testing.common import ( + create_fleet, + create_project, + create_user, +) async def _create_endpoint_model( session: AsyncSession, - max_agent_budget: float | None = None, max_price: float | None = None, spot_policy: SpotPolicy | None = None, backends: list[BackendType] | None = None, fleets: list[str] | None = None, + create_default_fleet: bool = True, ) -> EndpointModel: user = await create_user(session=session, name="admin", token="user-token") project = await create_project(session=session, owner=user, name="main") + if create_default_fleet: + await create_fleet(session=session, project=project) configuration = EndpointConfiguration( name="qwen-endpoint", model="Qwen/Qwen3-0.6B", env=Env.parse_obj({"HF_TOKEN": "hf-secret"}), - max_agent_budget=max_agent_budget, max_price=max_price, spot_policy=spot_policy, backends=backends, @@ -89,34 +99,348 @@ async def test_default_runner_starts_agent_detached( _configure_fake_claude(tmp_path, monkeypatch) monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") started = {} + written_logs = [] + + def write_logs(**kwargs): + written_logs.extend(log.message.decode().rstrip("\n") for log in kwargs["job_logs"]) def start_agent(workspace, request): started["workspace"] = workspace started["request"] = request return 12345 + monkeypatch.setattr(claude_module.logs_services, "write_logs", write_logs) monkeypatch.setattr(claude_module, "_start_agent_subprocess_detached", start_agent) - endpoint_model = await _create_endpoint_model(session, max_agent_budget=1.5) + endpoint_model = await _create_endpoint_model(session) service = ClaudeAgentService(workspace_base_dir=tmp_path) result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) assert result.in_progress is True assert result.error is None - attempt_root = tmp_path / str(endpoint_model.id) / "1" - assert started["workspace"].root_dir == attempt_root - assert started["request"]["cwd"] == str(attempt_root / "workspace") - assert started["request"]["options"]["max_budget"] == 1.5 + session_root = tmp_path / str(endpoint_model.id) / "1" + assert started["workspace"].root_dir == session_root + assert started["request"]["cwd"] == str(session_root / "workspace") + assert "max_budget" not in started["request"]["options"] res = await session.execute( - select(EndpointAgentAttemptModel).where( - EndpointAgentAttemptModel.endpoint_id == endpoint_model.id + select(EndpointAgentSessionModel).where( + EndpointAgentSessionModel.endpoint_id == endpoint_model.id ) ) - attempt = res.scalar_one() - assert attempt.attempt_num == 1 - assert attempt.status == EndpointAgentAttemptStatus.RUNNING - assert attempt.pid == 12345 - assert attempt.workspace_path == str(attempt_root) + agent_session = res.scalar_one() + assert agent_session.session_num == 1 + assert agent_session.status == EndpointAgentSessionStatus.RUNNING + assert agent_session.pid == 12345 + assert agent_session.workspace_path == str(session_root) + progress_path = session_root / "workspace" / "progress.jsonl" + progress_text = progress_path.read_text() + assert "Starting endpoint prototyping agent for Qwen/Qwen3-0.6B" in progress_text + assert agent_session.progress_log_offset == progress_path.stat().st_size + assert written_logs == [ + "Starting endpoint prototyping agent for Qwen/Qwen3-0.6B. " + "Allowed fleets: test-fleet. The agent will inspect offers, choose a service " + "recipe, deploy it, and verify the model API before the endpoint becomes active." + ] + + @pytest.mark.asyncio + async def test_restart_resumes_same_session_when_previous_process_exited( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") + endpoint_model = await _create_endpoint_model(session) + session_root = tmp_path / str(endpoint_model.id) / "1" + work_dir = session_root / "workspace" + work_dir.mkdir(parents=True) + (work_dir / "submissions.jsonl").write_text( + json.dumps( + { + "name": "qwen-endpoint-1", + "type": "service", + "status": "submitted", + "run_id": str(uuid.uuid4()), + } + ) + + "\n", + encoding="utf-8", + ) + agent_session = EndpointAgentSessionModel( + endpoint_id=endpoint_model.id, + session_num=1, + status=EndpointAgentSessionStatus.RUNNING, + workspace_path=str(session_root), + pid=12345, + process_host=claude_module._get_process_host(), + progress_log_offset=0, + stdout_log_offset=0, + stderr_log_offset=0, + created_at=endpoint_model.created_at, + updated_at=endpoint_model.created_at, + ) + session.add(agent_session) + await session.commit() + monkeypatch.setattr(claude_module, "_is_process_group_running", lambda pgid: False) + started = {} + + def start_agent(workspace, request): + started["workspace"] = workspace + started["request"] = request + return 23456 + + monkeypatch.setattr(claude_module, "_start_agent_subprocess_detached", start_agent) + service = ClaudeAgentService(workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.in_progress is True + assert result.error is None + assert started["workspace"].root_dir == session_root + assert "On startup or resume" in started["request"]["prompt"] + assert "previous_sessions.md" not in started["request"]["prompt"] + await session.refresh(agent_session) + assert agent_session.session_num == 1 + assert agent_session.status == EndpointAgentSessionStatus.RUNNING + assert agent_session.pid == 23456 + assert agent_session.workspace_path == str(session_root) + + @pytest.mark.asyncio + async def test_new_endpoint_lifecycle_starts_new_session_without_previous_context( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + endpoint_model = await _create_endpoint_model(session) + previous_root = tmp_path / str(endpoint_model.id) / "1" + previous_work_dir = previous_root / "workspace" + previous_work_dir.mkdir(parents=True) + previous_run_id = uuid.uuid4() + (previous_work_dir / "final_report.json").write_text( + json.dumps( + { + "success": False, + "run_id": str(previous_run_id), + "run_name": "qwen-endpoint-1", + "service_yaml": "type: service\nname: qwen-endpoint-1\n", + "failure_summary": "Previous image required a newer driver.", + } + ) + + "\n", + encoding="utf-8", + ) + (previous_work_dir / "submissions.jsonl").write_text( + json.dumps( + { + "name": "qwen-endpoint-1", + "status": "failed", + "run_id": str(previous_run_id), + } + ) + + "\n", + encoding="utf-8", + ) + previous_session = EndpointAgentSessionModel( + endpoint_id=endpoint_model.id, + session_num=1, + status=EndpointAgentSessionStatus.FAILED, + workspace_path=str(previous_root), + progress_log_offset=0, + stdout_log_offset=0, + stderr_log_offset=0, + status_message="Previous image required a newer driver.", + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + updated_at=datetime(2023, 1, 2, 3, 5, tzinfo=timezone.utc), + finished_at=datetime(2023, 1, 2, 3, 6, tzinfo=timezone.utc), + ) + session.add(previous_session) + endpoint_model.created_at = datetime(2023, 1, 3, 3, 4, tzinfo=timezone.utc) + await session.commit() + captured = {} + + async def runner(workspace, request): + captured["workspace"] = workspace + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=uuid.uuid4(), + run_name="qwen-endpoint-1", + service_yaml="type: service\nname: qwen-endpoint-1\n", + ) + ) + + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + assert captured["workspace"].root_dir == tmp_path / str(endpoint_model.id) / "2" + prompt = captured["request"]["prompt"] + assert "previous_sessions.md" not in prompt + assert "Previous image required a newer driver." not in prompt + assert not ( + tmp_path / str(endpoint_model.id) / "2" / "workspace" / "previous_sessions.md" + ).exists() + res = await session.execute( + select(EndpointAgentSessionModel) + .where(EndpointAgentSessionModel.endpoint_id == endpoint_model.id) + .order_by(EndpointAgentSessionModel.session_num) + ) + sessions = list(res.scalars().all()) + assert [agent_session.session_num for agent_session in sessions] == [1, 2] + + @pytest.mark.asyncio + async def test_abort_endpoint_stops_agent_process_group( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") + endpoint_model = await _create_endpoint_model(session) + session_root = tmp_path / str(endpoint_model.id) / "1" + work_dir = session_root / "workspace" + work_dir.mkdir(parents=True) + process_state_path = work_dir / "agent_process.json" + process_state_path.write_text( + json.dumps( + { + "pid": 12345, + "pgid": 12345, + "host": claude_module._get_process_host(), + } + ) + + "\n", + encoding="utf-8", + ) + agent_session = EndpointAgentSessionModel( + endpoint_id=endpoint_model.id, + session_num=1, + status=EndpointAgentSessionStatus.RUNNING, + workspace_path=str(session_root), + pid=12345, + process_host=claude_module._get_process_host(), + progress_log_offset=0, + stdout_log_offset=0, + stderr_log_offset=0, + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + updated_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + session.add(agent_session) + await session.commit() + + group_running = iter([True, False]) + signals = [] + + monkeypatch.setattr( + claude_module, + "_is_process_group_running", + lambda pgid: next(group_running), + ) + monkeypatch.setattr(claude_module, "_AGENT_PROCESS_ABORT_GRACE_SECONDS", 0) + monkeypatch.setattr( + claude_module.os, + "killpg", + lambda pgid, sig: signals.append((pgid, sig)), + ) + + service = ClaudeAgentService(workspace_base_dir=tmp_path) + + aborted = await service.abort_endpoint(endpoint_model) + + assert aborted is True + assert signals == [(12345, signal.SIGTERM)] + assert not process_state_path.exists() + await session.refresh(agent_session) + assert agent_session.status == EndpointAgentSessionStatus.FAILED + assert agent_session.status_message == "Endpoint stop requested" + + @pytest.mark.asyncio + async def test_abort_endpoint_waits_for_agent_process_on_another_host( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") + endpoint_model = await _create_endpoint_model(session) + session_root = tmp_path / str(endpoint_model.id) / "1" + work_dir = session_root / "workspace" + work_dir.mkdir(parents=True) + process_state_path = work_dir / "agent_process.json" + process_state_path.write_text( + json.dumps({"pid": 12345, "pgid": 12345, "host": "other-host"}) + "\n", + encoding="utf-8", + ) + agent_session = EndpointAgentSessionModel( + endpoint_id=endpoint_model.id, + session_num=1, + status=EndpointAgentSessionStatus.RUNNING, + workspace_path=str(session_root), + pid=12345, + process_host="other-host", + progress_log_offset=0, + stdout_log_offset=0, + stderr_log_offset=0, + created_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + updated_at=datetime(2023, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + session.add(agent_session) + await session.commit() + monkeypatch.setattr( + claude_module.os, + "killpg", + lambda pgid, sig: pytest.fail("remote process group must not be signaled locally"), + ) + + service = ClaudeAgentService(workspace_base_dir=tmp_path) + + aborted = await service.abort_endpoint(endpoint_model) + + assert aborted is False + assert process_state_path.exists() + await session.refresh(agent_session) + assert agent_session.status == EndpointAgentSessionStatus.RUNNING + + def test_process_group_liveness_ignores_zombie_only_group(self, monkeypatch): + class _PsResult: + returncode = 0 + stdout = "64979 Z\n64979 Z+\n" + + def killpg(pgid, sig): + raise PermissionError + + monkeypatch.setattr(claude_module.os, "killpg", killpg) + monkeypatch.setattr( + claude_module.subprocess, + "run", + lambda *args, **kwargs: _PsResult(), + ) + + assert claude_module._is_process_group_running(64979) is False + + def test_process_group_liveness_detects_non_zombie_member(self, monkeypatch): + class _PsResult: + returncode = 0 + stdout = "64979 Z\n64979 S+\n" + + def killpg(pgid, sig): + raise PermissionError + + monkeypatch.setattr(claude_module.os, "killpg", killpg) + monkeypatch.setattr( + claude_module.subprocess, + "run", + lambda *args, **kwargs: _PsResult(), + ) + + assert claude_module._is_process_group_running(64979) is True @pytest.mark.asyncio async def test_invokes_agent_with_isolated_dstack_cli_context( @@ -128,7 +452,6 @@ async def test_invokes_agent_with_isolated_dstack_cli_context( monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") claude_path = _configure_fake_claude(tmp_path, monkeypatch) monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MODEL", "test-claude-model") - monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MAX_BUDGET", 3.0) monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") monkeypatch.setenv("DATABASE_URL", "must-not-leak") captured = {} @@ -141,36 +464,42 @@ async def runner(workspace, request): report=AgentFinalReport( success=True, run_id=run_id, - run_name="qwen-agent-candidate", - service_yaml="type: service\nname: qwen-agent-candidate\n", - verification_summary="Agent verified chat completions.", + run_name="qwen-endpoint-1", + service_yaml="type: service\nname: qwen-endpoint-1\n", ) ) - endpoint_model = await _create_endpoint_model(session, max_agent_budget=1.5) + endpoint_model = await _create_endpoint_model(session) service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) assert result.error is None assert result.run_id == run_id - assert result.run_name == "qwen-agent-candidate" + assert result.run_name == "qwen-endpoint-1" assert result.final_report is not None - assert result.final_report.verification_summary == "Agent verified chat completions." + assert result.final_report.success is True assert ( - "successful final report must include the verified service run `run_id`" + "`final_report.json` must contain only the schema fields" in captured["request"]["prompt"] ) - attempt_root = tmp_path / str(endpoint_model.id) / "1" - assert captured["request"]["cwd"] == str(attempt_root / "workspace") + session_root = tmp_path / str(endpoint_model.id) / "1" + assert captured["request"]["cwd"] == str(session_root / "workspace") env = captured["request"]["env"] assert env["ANTHROPIC_API_KEY"] == "agent-secret" - assert env["HOME"] == str(attempt_root / "home") + agent_home = Path(env["HOME"]) + assert agent_home != session_root / "home" + assert agent_home.resolve() == session_root / "home" + control_sock_path = agent_home / ".dstack" / "ssh" / f"{'x' * 60}.control.sock" + assert len(str(control_sock_path)) < 104 + assert stat.S_IMODE((session_root / "home" / ".ssh").stat().st_mode) == 0o700 assert env["DSTACK_PROJECT"] == "main" + assert env["DSTACK_ENDPOINT_SERVER_URL"] == "http://127.0.0.1:8000" + assert env["DSTACK_ENDPOINT_BEARER_TOKEN"] == "user-token" assert env["HF_TOKEN"] == "hf-secret" assert "DATABASE_URL" not in env assert captured["request"]["options"]["model"] == "test-claude-model" - assert captured["request"]["options"]["max_budget"] == 1.5 + assert "max_budget" not in captured["request"]["options"] assert "StructuredOutput" in captured["request"]["options"]["tools"] assert "Edit" in captured["request"]["options"]["allowed_tools"] assert "StructuredOutput" in captured["request"]["options"]["allowed_tools"] @@ -182,9 +511,8 @@ async def runner(workspace, request): assert command[command.index("--tools") + 1] == captured["request"]["options"]["tools"] assert "--permission-mode" in command assert command[command.index("--permission-mode") + 1] == "bypassPermissions" - assert "--max-budget-usd" in command - assert command[command.index("--max-budget-usd") + 1] == "1.5" - config = yaml.safe_load((attempt_root / "home" / ".dstack" / "config.yml").read_text()) + assert "--max-budget-usd" not in command + config = yaml.safe_load((session_root / "home" / ".dstack" / "config.yml").read_text()) assert config == { "projects": [ { @@ -195,22 +523,37 @@ async def runner(workspace, request): } ], } - work_dir = attempt_root / "workspace" + work_dir = session_root / "workspace" state = json.loads((work_dir / "agent_state.json").read_text()) assert state["endpoint_name"] == "qwen-endpoint" assert state["model"] == "Qwen/Qwen3-0.6B" assert state["phase"] == "success" - assert state["max_agent_budget"] == 1.5 - assert (work_dir / "sources.jsonl").exists() - assert (work_dir / "candidates.jsonl").exists() + assert "max_agent_budget" not in state + assert not (work_dir / "sources.jsonl").exists() + assert (work_dir / "submissions.jsonl").exists() assert (work_dir / "commands.jsonl").exists() assert (work_dir / "progress.jsonl").exists() - assert (work_dir / "hardware_reasoning.md").exists() + progress_helper = session_root / "bin" / "progress" + assert progress_helper.exists() + assert not (session_root / "bin" / "dstack").exists() + progress_result = subprocess.run( + [str(progress_helper), "Checked model config."], + cwd=work_dir, + env=captured["request"]["env"], + capture_output=True, + text=True, + check=False, + ) + assert progress_result.returncode == 0 + assert json.loads((work_dir / "progress.jsonl").read_text().splitlines()[-1]) == { + "message": "Checked model config." + } + assert not (work_dir / "hardware_reasoning.md").exists() assert (work_dir / ".claude" / "skills" / "dstack" / "SKILL.md").exists() assert (work_dir / ".claude" / "skills" / "dstack-prototyping" / "SKILL.md").exists() final_report = json.loads((work_dir / "final_report.json").read_text()) assert final_report["success"] is True - assert final_report["run_name"] == "qwen-agent-candidate" + assert final_report["run_name"] == "qwen-endpoint-1" @pytest.mark.asyncio async def test_includes_endpoint_profile_constraints_in_prompt( @@ -229,9 +572,8 @@ async def runner(workspace, request): report=AgentFinalReport( success=True, run_id=uuid.uuid4(), - run_name="qwen-agent-candidate", - service_yaml="type: service\nname: qwen-agent-candidate\n", - verification_summary="Agent verified chat completions.", + run_name="qwen-endpoint-1", + service_yaml="type: service\nname: qwen-endpoint-1\n", ) ) @@ -248,130 +590,11 @@ async def runner(workspace, request): prompt = captured["request"]["prompt"] assert "- max_price: 0.3" in prompt assert "- spot_policy: on-demand" in prompt - assert "--max-price 0.3 --on-demand" in prompt + assert "Reuse these CLI flags" not in prompt + assert "--max-price 0.3 --on-demand" not in prompt assert "--availability-zone" not in prompt assert "--instance " not in prompt - @pytest.mark.asyncio - async def test_prompt_points_to_bundled_skills_and_endpoint_contract( - self, - session: AsyncSession, - tmp_path, - monkeypatch, - ): - monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") - _configure_fake_claude(tmp_path, monkeypatch) - captured = {} - - async def runner(workspace, request): - captured["request"] = request - return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=uuid.uuid4(), - run_name="qwen-agent-candidate", - service_yaml="type: service\nname: qwen-agent-candidate\n", - verification_summary="Agent verified chat completions.", - ) - ) - - endpoint_model = await _create_endpoint_model( - session, - max_price=0.5, - spot_policy=SpotPolicy.ONDEMAND, - backends=[BackendType.RUNPOD], - fleets=["endpoint-e2e-runpod"], - ) - service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) - - result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) - - assert result.error is None - prompt = captured["request"]["prompt"] - assert "Load and follow `/dstack`" in prompt - assert "Load and follow `/dstack-prototyping`" in prompt - assert "Bundled Claude Code skills are installed in `.claude/skills`" in prompt - assert "Do not call hidden server APIs" in prompt - assert "real model API request" in prompt - assert "dstack-development" not in prompt - assert "progress.jsonl" in prompt - assert ( - "Do not write YAML, command output, long tables, raw traces, or secrets to " - "`progress.jsonl`" in prompt - ) - assert ( - "Record each dev environment, task, or service candidate in `candidates.jsonl`" - in prompt - ) - assert "verification.json" in prompt - assert "service run name must not equal the endpoint name `qwen-endpoint`" in prompt - assert "prefer `qwen-endpoint-1`, `qwen-endpoint-2`, etc." in prompt - assert "Make each service YAML self-contained" in prompt - assert "Do not wait only for log text" in prompt - assert "Use normal service logs first" in prompt - assert "- backends: runpod" in prompt - assert ( - "- Reuse these CLI flags where applicable: --max-price 0.5 --on-demand --backend runpod --fleet endpoint-e2e-runpod" - in prompt - ) - assert "RUNPOD" not in prompt - work_dir = tmp_path / str(endpoint_model.id) / "1" / "workspace" - prototyping_skill = ( - work_dir / ".claude" / "skills" / "dstack-prototyping" / "SKILL.md" - ).read_text() - assert "Load `/dstack` first" in prototyping_skill - assert "Do not repeat or override `/dstack` command syntax here" in prototyping_skill - assert "Start with vLLM and SGLang" in prototyping_skill - assert "Use a dev environment when an interactive shell can answer" in prototyping_skill - assert "Never collapse tested hardware back into minimum requirements" in prototyping_skill - assert "Do not treat `running`, a passed service probe, or clean logs" in prototyping_skill - assert ( - "Retrying the same YAML after the same error is not prototyping" in prototyping_skill - ) - assert "do not name a candidate service exactly like the endpoint" in prototyping_skill - assert "Do not use `:latest` for a final serving image" in prototyping_skill - assert "poll run JSON and stop waiting on terminal states" in prototyping_skill - assert "Use normal logs before diagnostic logs" in prototyping_skill - assert "include applicable backend, fleet, price, spot" in prototyping_skill - assert "https://recipes.vllm.ai/models.json" in prototyping_skill - assert ( - "https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development" - in prototyping_skill - ) - assert "dstack-development" not in prototyping_skill - - @pytest.mark.asyncio - async def test_uses_server_default_agent_budget_when_endpoint_does_not_set_one( - self, - session: AsyncSession, - tmp_path, - monkeypatch, - ): - monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") - _configure_fake_claude(tmp_path, monkeypatch) - monkeypatch.setattr(settings, "AGENT_ANTHROPIC_MAX_BUDGET", 4.0) - captured = {} - - async def runner(workspace, request): - captured["request"] = request - return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=uuid.uuid4(), - run_name="qwen-agent-candidate", - service_yaml="type: service\nname: qwen-agent-candidate\n", - verification_summary="Agent verified chat completions.", - ) - ) - - endpoint_model = await _create_endpoint_model(session) - service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) - - result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) - - assert result.error is None - assert captured["request"]["options"]["max_budget"] == 4.0 - @pytest.mark.asyncio async def test_reuses_existing_final_report_without_invoking_runner( self, @@ -390,9 +613,8 @@ async def test_reuses_existing_final_report_without_invoking_runner( { "success": True, "run_id": str(run_id), - "run_name": "qwen-agent-candidate", - "service_yaml": "type: service\nname: qwen-agent-candidate\n", - "verification_summary": "Verified before restart.", + "run_name": "qwen-endpoint-1", + "service_yaml": "type: service\nname: qwen-endpoint-1\n", } ), encoding="utf-8", @@ -408,9 +630,9 @@ def start_agent(workspace, request): assert result.error is None assert result.run_id == run_id - assert result.run_name == "qwen-agent-candidate" + assert result.run_name == "qwen-endpoint-1" assert result.final_report is not None - assert result.final_report.verification_summary == "Verified before restart." + assert result.final_report.success is True @pytest.mark.asyncio async def test_waits_for_claude_result_after_final_report_while_process_runs( @@ -424,7 +646,7 @@ async def test_waits_for_claude_result_after_final_report_while_process_runs( monkeypatch.setattr( claude_module, "_get_running_agent_process_pid", - lambda workspace, attempt=None: 123, + lambda workspace, agent_session=None: 123, ) endpoint_model = await _create_endpoint_model(session) work_dir = tmp_path / str(endpoint_model.id) / "workspace" @@ -434,9 +656,8 @@ async def test_waits_for_claude_result_after_final_report_while_process_runs( { "success": True, "run_id": str(uuid.uuid4()), - "run_name": "qwen-agent-candidate", - "service_yaml": "type: service\nname: qwen-agent-candidate\n", - "verification_summary": "Verified before Claude exited.", + "run_name": "qwen-endpoint-1", + "service_yaml": "type: service\nname: qwen-endpoint-1\n", } ), encoding="utf-8", @@ -449,57 +670,6 @@ async def test_waits_for_claude_result_after_final_report_while_process_runs( assert result.in_progress is True assert result.final_report is None - @pytest.mark.asyncio - async def test_reconciles_claude_cost_from_result_stream( - self, - session: AsyncSession, - tmp_path, - monkeypatch, - ): - monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") - _configure_fake_claude(tmp_path, monkeypatch) - endpoint_model = await _create_endpoint_model(session) - work_dir = tmp_path / str(endpoint_model.id) / "workspace" - work_dir.mkdir(parents=True) - run_id = uuid.uuid4() - (work_dir / "final_report.json").write_text( - json.dumps( - { - "success": True, - "run_id": str(run_id), - "run_name": "qwen-agent-candidate", - "service_yaml": "type: service\nname: qwen-agent-candidate\n", - "verification_summary": "Verified before restart.", - } - ), - encoding="utf-8", - ) - (work_dir / "agent_stdout.jsonl").write_text( - json.dumps( - { - "type": "result", - "is_error": False, - "total_cost_usd": 0.42, - } - ) - + "\n", - encoding="utf-8", - ) - service = ClaudeAgentService(workspace_base_dir=tmp_path) - - result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) - - assert result.error is None - assert result.run_id == run_id - res = await session.execute( - select(EndpointAgentAttemptModel).where( - EndpointAgentAttemptModel.endpoint_id == endpoint_model.id - ) - ) - attempt = res.scalar_one() - assert attempt.status == EndpointAgentAttemptStatus.SUCCEEDED - assert attempt.spent_agent_budget == 0.42 - @pytest.mark.asyncio async def test_returns_in_progress_when_agent_process_is_still_running( self, @@ -512,7 +682,7 @@ async def test_returns_in_progress_when_agent_process_is_still_running( monkeypatch.setattr( claude_module, "_get_running_agent_process_pid", - lambda workspace, attempt=None: 123, + lambda workspace, agent_session=None: 123, ) endpoint_model = await _create_endpoint_model(session) @@ -549,9 +719,8 @@ async def runner(workspace, request): report=AgentFinalReport( success=True, run_id=run_id, - run_name="qwen-agent-candidate", + run_name="qwen-endpoint-1", service_yaml="token: hf-secret\n", - verification_summary="Agent verified with agent-secret.", ) ) @@ -581,7 +750,7 @@ async def test_returns_error_when_sdk_fails_before_report( async def runner(workspace, request): return _AgentRunnerResult( - error="Server agent failed before returning a verification report: bad key" + error="Server agent failed before returning a final report: bad key" ) endpoint_model = await _create_endpoint_model(session) @@ -589,9 +758,7 @@ async def runner(workspace, request): result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) - assert ( - result.error == "Server agent failed before returning a verification report: bad key" - ) + assert result.error == "Server agent failed before returning a final report: bad key" assert result.final_report is None work_dir = tmp_path / str(endpoint_model.id) / "1" / "workspace" state = json.loads((work_dir / "agent_state.json").read_text()) @@ -599,7 +766,7 @@ async def runner(workspace, request): agent_error = json.loads((work_dir / "agent_error.json").read_text()) assert agent_error["success"] is False assert agent_error["failure_summary"] == ( - "Server agent failed before returning a verification report: bad key" + "Server agent failed before returning a final report: bad key" ) def test_returns_error_when_configured_claude_path_is_not_executable( @@ -639,7 +806,6 @@ def test_falls_back_to_claude_in_path(self, monkeypatch: pytest.MonkeyPatch): "disallowed_tools": "", "model": "test-model", "max_turns": 1, - "max_budget": None, "json_schema": {}, }, } @@ -660,14 +826,12 @@ async def test_reads_claude_stream_incrementally(self, tmp_path): redacted_values=["secret"], endpoint_name="qwen-endpoint", model="Qwen/Qwen3-0.6B", - max_agent_budget=None, ) report = { "success": True, "run_id": str(uuid.uuid4()), - "run_name": "qwen-agent-candidate", + "run_name": "qwen-endpoint-1", "service_yaml": "env: secret\n", - "verification_summary": "verified secret", } reader.feed_data( json.dumps({"type": "assistant", "message": {"content": "working"}}).encode() + b"\n" @@ -684,8 +848,41 @@ async def test_reads_claude_stream_incrementally(self, tmp_path): assert "secret" not in trace assert "[redacted]" in trace + def test_loads_submitted_run_names_without_run_ids(self, tmp_path): + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=tmp_path / "trace.jsonl", + env={}, + redacted_values=[], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + ) + workspace.work_dir.mkdir(parents=True) + run_id = uuid.uuid4() + (workspace.work_dir / "submissions.jsonl").write_text( + "\n".join( + [ + json.dumps({"name": "qwen-endpoint-1", "run_id": None}), + json.dumps({"name": "qwen-endpoint-1", "run_id": str(run_id)}), + json.dumps({"name": "qwen-endpoint-2"}), + ] + ) + + "\n", + encoding="utf-8", + ) + + submissions = _load_submissions(workspace) + + assert submissions.run_ids == (run_id,) + assert submissions.run_names == ("qwen-endpoint-1", "qwen-endpoint-2") + @pytest.mark.asyncio - async def test_records_commands_without_copying_claude_stream_to_endpoint_logs(self, tmp_path): + async def test_stores_assistant_text_without_copying_stream_to_endpoint_logs( + self, + tmp_path, + ): class FakeLogWriter: def __init__(self): self.messages = [] @@ -707,7 +904,6 @@ async def flush(self): redacted_values=["secret"], endpoint_name="qwen-endpoint", model="Qwen/Qwen3-0.6B", - max_agent_budget=None, log_writer=log_writer, ) reader.feed_data( @@ -716,6 +912,10 @@ async def flush(self): "type": "assistant", "message": { "content": [ + { + "type": "text", + "text": "Checking offers without exposing secret.", + }, { "type": "tool_use", "id": "tool-1", @@ -724,7 +924,7 @@ async def flush(self): "description": "List offers", "command": "dstack offer --gpu 1 --max-price 0.3", }, - } + }, ] }, } @@ -788,7 +988,6 @@ async def flush(self): redacted_values=[], endpoint_name="qwen-endpoint", model="Qwen/Qwen3-0.6B", - max_agent_budget=None, log_writer=log_writer, ) long_output = "\n".join(f"offer {i:04d} gpu=A5000 price=0.27" for i in range(200)) @@ -841,15 +1040,14 @@ async def flush(self): claude_path = tmp_path / "claude" claude_path.write_text( """#!/bin/sh -printf '%s\\n' '{"phase":"research","message":"Checking recipes with secret"}' >> progress.jsonl -printf '%s\\n' '{"phase":"submit","message":"Submitted service candidate"}' >> progress.jsonl +printf '%s\\n' 'Checking recipes with secret' >> progress.jsonl +printf '%s\\n' '{"phase":"submit","message":"Submitted service run"}' >> progress.jsonl cat > final_report.json <<'JSON' { "success": true, "run_id": "6e578748-d597-4fde-a3a4-203587cad5a2", - "run_name": "qwen-agent-candidate", - "service_yaml": "type: service\\nname: qwen-agent-candidate\\n", - "verification_summary": "Verified chat completions." + "run_name": "qwen-endpoint-1", + "service_yaml": "type: service\\nname: qwen-endpoint-1\\n" } JSON printf '%s\\n' '{"type":"result","is_error":false,"result":"done"}' @@ -870,7 +1068,6 @@ async def flush(self): redacted_values=["secret"], endpoint_name="qwen-endpoint", model="Qwen/Qwen3-0.6B", - max_agent_budget=None, log_writer=log_writer, ) request = { @@ -882,7 +1079,6 @@ async def flush(self): "disallowed_tools": "", "model": "test-model", "max_turns": 1, - "max_budget": None, "json_schema": {}, }, } @@ -895,8 +1091,8 @@ async def flush(self): assert result.error is None assert log_writer.messages == [ - "research: Checking recipes with [redacted]", - "submit: Submitted service candidate", + "Checking recipes with [redacted]", + "Submitted service run", ] @pytest.mark.asyncio @@ -912,9 +1108,8 @@ async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missi { "success": true, "run_id": "6e578748-d597-4fde-a3a4-203587cad5a2", - "run_name": "qwen-agent-candidate", - "service_yaml": "type: service\\nname: qwen-agent-candidate\\n", - "verification_summary": "Verified chat completions over SSH." + "run_name": "qwen-endpoint-1", + "service_yaml": "type: service\\nname: qwen-endpoint-1\\n" } JSON printf '%s\\n' '{"type":"result","is_error":false,"result":"done"}' @@ -933,7 +1128,6 @@ async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missi redacted_values=[], endpoint_name="qwen-endpoint", model="Qwen/Qwen3-0.6B", - max_agent_budget=None, ) request = { "prompt": "prompt", @@ -944,7 +1138,6 @@ async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missi "disallowed_tools": "", "model": "test-model", "max_turns": 1, - "max_budget": None, "json_schema": {}, }, } @@ -954,8 +1147,7 @@ async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missi assert result.error is None assert result.report is not None assert result.report.success is True - assert result.report.run_name == "qwen-agent-candidate" - assert result.report.verification_summary == "Verified chat completions over SSH." + assert result.report.run_name == "qwen-endpoint-1" @pytest.mark.asyncio async def test_subprocess_no_report_error_does_not_include_stream_output( @@ -982,7 +1174,6 @@ async def test_subprocess_no_report_error_does_not_include_stream_output( redacted_values=[], endpoint_name="qwen-endpoint", model="Qwen/Qwen3-0.6B", - max_agent_budget=None, ) request = { "prompt": "prompt", @@ -992,7 +1183,6 @@ async def test_subprocess_no_report_error_does_not_include_stream_output( "disallowed_tools": "", "model": "test-model", "max_turns": 1, - "max_budget": None, "json_schema": {}, }, } @@ -1000,6 +1190,6 @@ async def test_subprocess_no_report_error_does_not_include_stream_output( result = await _run_agent_in_subprocess(workspace, request) assert result.error == ( - "Server agent process exited without a verification report (return code 143)" + "Server agent process exited without a final report (return code 143)" ) assert "huge offer table" not in result.error diff --git a/src/tests/_internal/server/services/test_endpoint_presets.py b/src/tests/_internal/server/services/test_endpoint_presets.py index 8f2369f443..2b9a104280 100644 --- a/src/tests/_internal/server/services/test_endpoint_presets.py +++ b/src/tests/_internal/server/services/test_endpoint_presets.py @@ -31,8 +31,11 @@ ) from dstack._internal.server.services.endpoints.presets import ( EndpointPreset, - EndpointPresetReplicaSpecGroup, + EndpointPresetRecipe, + EndpointPresetValidation, + EndpointPresetValidationReplica, LocalDirEndpointPresetService, + make_endpoint_preset_recipe_id, ) from dstack._internal.server.testing.common import ( create_job, @@ -55,6 +58,109 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): """\ model: Qwen/Qwen3-4B type: endpoint-preset +recipes: + - id: vllm-t4 + service: + image: vllm/vllm-openai:latest + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""" + ) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) + + assert len(presets) == 1 + assert presets[0].model == "Qwen/Qwen3-4B" + assert [recipe.id for recipe in presets[0].recipes] == ["vllm-t4"] + recipe = presets[0].recipes[0] + assert recipe.service.resources.gpu is not None + assert recipe.service.resources.gpu.vendor is None + assert recipe.validations[0].replicas[0].resources[0].gpu is not None + + @pytest.mark.asyncio + async def test_lists_replica_group_presets_in_group_order(self, tmp_path): + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +recipes: + - id: grouped + service: + port: 8000 + model: Qwen/Qwen3-4B + replicas: + - name: router + count: 1 + image: ghcr.io/example/router:latest + commands: + - python router.py + resources: + cpu: 4 + - name: worker + count: 2 + image: vllm/vllm-openai:latest + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + resources: + gpu: 24GB + validations: + - replicas: + - resources: + - cpu: 8 + memory: 16GB + disk: 100GB + gpu: 0 + - resources: + - cpu: 14 + memory: 64GB + disk: 200GB + gpu: + name: L4 + memory: 24GB + count: 1 + - cpu: 14 + memory: 64GB + disk: 200GB + gpu: + name: L4 + memory: 24GB + count: 1 +""" + ) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) + + assert len(presets) == 1 + replica_groups = presets[0].recipes[0].service.replica_groups + assert [group.name for group in replica_groups] == ["router", "worker"] + assert replica_groups[0].resources.gpu is not None + assert replica_groups[0].resources.gpu.count.min == 0 + assert replica_groups[0].resources.cpu.count.min == 4 + assert replica_groups[1].resources.gpu is not None + + @pytest.mark.asyncio + async def test_loads_legacy_replica_spec_groups_as_recipe_validation(self, tmp_path): + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen-legacy.yml").write_text( + """\ +model: Qwen/Qwen3-4B +type: endpoint-preset service: image: vllm/vllm-openai:latest commands: @@ -77,80 +183,49 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert len(presets) == 1 - assert presets[0].name == "qwen" - assert presets[0].model == "Qwen/Qwen3-4B" - assert len(presets[0].replica_spec_groups) == 1 - assert presets[0].replica_spec_groups[0].name == "0" - assert presets[0].replica_spec_groups[0].resources.gpu is not None - assert presets[0].replica_spec_groups[0].tested_resources[0].gpu is not None - assert presets[0].configuration.resources.gpu is not None + recipe = presets[0].recipes[0] + assert recipe.service.resources.gpu is not None + assert recipe.service.resources.gpu.vendor is None + assert recipe.validations[0].replicas[0].resources[0].gpu is not None @pytest.mark.asyncio - async def test_lists_replica_group_presets_in_group_order(self, tmp_path): + async def test_rejects_service_gpu_vendor_mismatch(self, tmp_path, caplog): presets_dir = _project_presets_dir(tmp_path) presets_dir.mkdir(parents=True) (presets_dir / "qwen.yml").write_text( """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - port: 8000 - model: Qwen/Qwen3-4B - replicas: - - name: router - count: 1 - image: ghcr.io/example/router:latest - commands: - - python router.py - - name: worker - count: 2 - image: vllm/vllm-openai:latest +recipes: + - id: vllm-amd + service: commands: - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 -replica_spec_groups: - - name: router - resources: - cpu: 4 - tested_resources: - - cpu: 8 - memory: 16GB - disk: 100GB - gpu: 0 - - name: worker - resources: - gpu: 24GB - tested_resources: - - cpu: 14 - memory: 64GB - disk: 200GB - gpu: - name: L4 - memory: 24GB - count: 1 - - cpu: 14 - memory: 64GB - disk: 200GB - gpu: - name: L4 - memory: 24GB - count: 1 + port: 8000 + resources: + gpu: amd:16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """ ) presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) - assert len(presets) == 1 - assert [group.name for group in presets[0].replica_spec_groups] == ["router", "worker"] - replica_groups = presets[0].configuration.replica_groups - assert [group.name for group in replica_groups] == ["router", "worker"] - assert replica_groups[0].resources.gpu is not None - assert replica_groups[0].resources.gpu.count.min == 0 - assert replica_groups[0].resources.cpu.count.min == 4 - assert replica_groups[1].resources.gpu is not None + assert presets == [] + assert "preset service GPU vendor does not match validation" in caplog.text @pytest.mark.asyncio - async def test_saves_preset_without_overwriting(self, tmp_path): - preset = _qwen_preset(name="qwen") + async def test_saves_preset_with_comments_and_merges_same_model(self, tmp_path): + preset = _qwen_preset(recipe_id="vllm-t4") + extra_preset = _qwen_preset(recipe_id="vllm-l4", resources={"gpu": "24GB"}) service = LocalDirEndpointPresetService(tmp_path) saved = await service.save_preset( @@ -161,52 +236,82 @@ async def test_saves_preset_without_overwriting(self, tmp_path): "run: 00000000-0000-0000-0000-000000000001", ], ) - saved_again = await service.save_preset(PROJECT_NAME, preset) - - assert saved.name == "qwen" - assert saved_again.name == "qwen-2" + assert saved.model == "Qwen/Qwen3-4B" presets_dir = _project_presets_dir(tmp_path) - assert (presets_dir / "qwen.dstack.yml").exists() - assert (presets_dir / "qwen-2.dstack.yml").exists() - text = (presets_dir / "qwen.dstack.yml").read_text() + preset_files = list(presets_dir.glob("*.dstack.yml")) + assert len(preset_files) == 1 + text = preset_files[0].read_text() assert text.startswith( "# endpoint: qwen-endpoint\n# run: 00000000-0000-0000-0000-000000000001\n" ) assert "SECRET_VALUE" not in text assert "preset-service-name" not in text assert "creation_policy" not in text - assert "tested_resources:" in text + assert "validations:" in text assert "HF_HOME=/root/.cache/huggingface" in text assert "HF_TOKEN" in text + saved_again = await service.save_preset(PROJECT_NAME, extra_preset) + + assert [recipe.id for recipe in saved_again.recipes] == ["vllm-l4", "vllm-t4"] + assert len(list(presets_dir.glob("*.dstack.yml"))) == 1 + + @pytest.mark.asyncio + async def test_merges_duplicate_model_files_for_list_get_save_and_delete(self, tmp_path): + service = LocalDirEndpointPresetService(tmp_path) + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + _write_qwen_preset_file(presets_dir / "qwen-a.yml", recipe_id="vllm-t4", gpu="16GB") + _write_qwen_preset_file(presets_dir / "qwen-b.yml", recipe_id="vllm-l4", gpu="24GB") + + presets = await service.list_presets(PROJECT_NAME) + preset = await service.get_preset(PROJECT_NAME, "Qwen/Qwen3-4B") + + assert len(presets) == 1 + assert preset is not None + assert [recipe.id for recipe in presets[0].recipes] == ["vllm-t4", "vllm-l4"] + assert [recipe.id for recipe in preset.recipes] == ["vllm-t4", "vllm-l4"] + + saved = await service.save_preset( + PROJECT_NAME, + _qwen_preset(recipe_id="vllm-a100", resources={"gpu": "80GB"}), + ) + + assert [recipe.id for recipe in saved.recipes] == ["vllm-a100", "vllm-t4", "vllm-l4"] + assert len(list(presets_dir.glob("*.yml"))) == 1 + + await service.delete_preset(PROJECT_NAME, "Qwen/Qwen3-4B") + + assert await service.list_presets(PROJECT_NAME) == [] + assert list(presets_dir.glob("*.yml")) == [] @pytest.mark.asyncio async def test_saved_preset_round_trips(self, tmp_path): saved = await LocalDirEndpointPresetService(tmp_path).save_preset( PROJECT_NAME, - _qwen_preset(name="Qwen/Qwen3-4B vLLM"), + _qwen_preset(recipe_id="vllm-t4"), ) presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) - assert saved.name == "qwen-qwen3-4b-vllm" + assert saved.model == "Qwen/Qwen3-4B" assert len(presets) == 1 - assert presets[0].name == saved.name assert presets[0].model == "Qwen/Qwen3-4B" - assert presets[0].configuration.model is not None - assert presets[0].configuration.model.name == "Qwen/Qwen3-4B" - assert set(presets[0].configuration.env.keys()) == {"HF_HOME", "HF_TOKEN"} - assert presets[0].configuration.env["HF_HOME"] == "/root/.cache/huggingface" - assert presets[0].configuration.resources.gpu is not None + recipe = presets[0].recipes[0] + assert recipe.service.model is not None + assert recipe.service.model.name == "Qwen/Qwen3-4B" + assert set(recipe.service.env.keys()) == {"HF_HOME", "HF_TOKEN"} + assert recipe.service.env["HF_HOME"] == "/root/.cache/huggingface" + assert recipe.service.resources.gpu is not None @pytest.mark.asyncio - async def test_deletes_preset_by_name(self, tmp_path): + async def test_deletes_preset_by_model(self, tmp_path): service = LocalDirEndpointPresetService(tmp_path) - saved = await service.save_preset(PROJECT_NAME, _qwen_preset(name="qwen")) + saved = await service.save_preset(PROJECT_NAME, _qwen_preset()) - await service.delete_preset(PROJECT_NAME, saved.name) + await service.delete_preset(PROJECT_NAME, saved.model) assert await service.list_presets(PROJECT_NAME) == [] - assert not (_project_presets_dir(tmp_path) / "qwen.dstack.yml").exists() + assert list(_project_presets_dir(tmp_path).glob("*.dstack.yml")) == [] @pytest.mark.asyncio async def test_delete_missing_preset_raises(self, tmp_path): @@ -238,186 +343,197 @@ async def test_ignores_non_yaml_files(self, tmp_path): "missing-model.yml", """\ type: endpoint-preset -service: - commands: - - python -m http.server 8000 - port: 8000 -replica_spec_groups: - - resources: - gpu: 16GB - tested_resources: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: 0 +recipes: + - id: vllm + service: + commands: + - python -m http.server 8000 + port: 8000 + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: 0 """, "preset must specify a model", ), ( - "mixed-resources.yml", + "missing-recipes.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - commands: - - python -m http.server 8000 - port: 8000 - resources: - gpu: 24GB -replica_spec_groups: - - resources: - gpu: 16GB - tested_resources: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 """, - "preset service object must not specify resources", + "preset recipe must specify a service object", ), ( - "service-profile.yml", + "missing-service.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - commands: - - python -m http.server 8000 - port: 8000 - creation_policy: reuse -replica_spec_groups: - - resources: - gpu: 16GB - tested_resources: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 +recipes: + - id: vllm + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: 0 """, - "preset service object must not specify profile fields", + "preset recipe must specify a service object", ), ( - "group-resources.yml", + "service-profile.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - port: 8000 - replicas: - - name: worker - count: 1 - image: vllm/vllm-openai:latest +recipes: + - id: vllm + service: + commands: + - python -m http.server 8000 + port: 8000 resources: - gpu: 24GB -replica_spec_groups: - - name: worker - replica_specs: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 + gpu: 16GB + creation_policy: reuse + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """, - "preset service replica groups must not specify resources", + "preset service object must not specify profile fields", ), ( - "missing-replica-spec-groups.yml", + "missing-service-resources.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - port: 8000 - replicas: - - name: worker - count: 1 - image: vllm/vllm-openai:latest +recipes: + - id: vllm + service: + commands: + - python -m http.server 8000 + port: 8000 + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """, - "preset must specify non-empty replica_spec_groups", + "preset service object must specify resources", ), ( - "loose-resources.yml", + "group-missing-resources.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - commands: - - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 - port: 8000 -replica_spec_groups: - - replica_specs: - - gpu: 16GB +recipes: + - id: vllm + service: + port: 8000 + replicas: + - name: worker + count: 1 + image: vllm/vllm-openai:latest + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """, - "preset tested_resources must use exact replica resources", + "preset service replica groups must specify resources", ), ( - "mismatched-replica-spec-groups.yml", + "loose-resources.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - port: 8000 - replicas: - - name: worker - count: 1 - image: vllm/vllm-openai:latest -replica_spec_groups: - - name: other - replica_specs: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 +recipes: + - id: vllm + service: + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - gpu: 16GB """, - "preset replica_spec_groups must match replica group order", + "preset validations must use exact replica resources", ), ( - "missing-tested-resources.yml", + "mismatched-validation-replicas.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - port: 8000 - replicas: - - name: worker - count: 1 - image: vllm/vllm-openai:latest -replica_spec_groups: - - name: worker - resources: - gpu: 16GB +recipes: + - id: grouped + service: + port: 8000 + replicas: + - name: router + count: 1 + image: ghcr.io/example/router:latest + resources: + cpu: 4 + - name: worker + count: 1 + image: vllm/vllm-openai:latest + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: 0 """, - "preset replica_spec_groups must specify resources and tested_resources", + "preset validation replicas must match service replica group order", ), ( "bad-replica-group-shape.yml", """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - port: 8000 - replicas: - - worker -replica_spec_groups: - - name: worker - replica_specs: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 +recipes: + - id: grouped + service: + port: 8000 + replicas: + - worker + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """, "preset service replica groups must be objects", ), @@ -480,16 +596,19 @@ async def test_builds_implicit_replica_group_from_running_service( ) await session.refresh(run, attribute_names=["jobs"]) - preset = build_endpoint_preset_from_run("qwen-learned", run) + preset = build_endpoint_preset_from_run(run) saved = await LocalDirEndpointPresetService(tmp_path).save_preset(PROJECT_NAME, preset) - assert saved.name == "qwen-learned" assert saved.model == "Qwen/Qwen3-4B" - assert [group.name for group in saved.replica_spec_groups] == ["0"] - assert "gpu=16GB" in saved.replica_spec_groups[0].resources.pretty_format() - tested_resources = saved.replica_spec_groups[0].tested_resources[0].pretty_format() - assert "cpu=14" in tested_resources - assert "gpu=L4:24GB:1" in tested_resources + recipe = saved.recipes[0] + assert recipe.id == make_endpoint_preset_recipe_id(recipe.service) + assert recipe.service.resources.gpu is not None + assert recipe.service.resources.gpu.vendor is not None + assert recipe.service.resources.gpu.vendor.value == "nvidia" + assert "gpu=nvidia:16GB" in recipe.service.resources.pretty_format() + resources = recipe.validations[0].replicas[0].resources[0].pretty_format() + assert "cpu=14" in resources + assert "gpu=L4:24GB:1" in resources @pytest.mark.asyncio async def test_requires_actual_instance_resources(self, session: AsyncSession): @@ -528,7 +647,7 @@ async def test_requires_actual_instance_resources(self, session: AsyncSession): await session.refresh(run, attribute_names=["jobs"]) with pytest.raises(ValueError, match="actual instance resources"): - build_endpoint_preset_from_run("qwen-learned", run) + build_endpoint_preset_from_run(run) @pytest.mark.asyncio async def test_builds_replica_groups_in_service_order(self, session: AsyncSession, tmp_path): @@ -603,18 +722,22 @@ async def test_builds_replica_groups_in_service_order(self, session: AsyncSessio ) await session.refresh(run, attribute_names=["jobs"]) - preset = build_endpoint_preset_from_run("qwen-grouped", run) + preset = build_endpoint_preset_from_run(run) saved = await LocalDirEndpointPresetService(tmp_path).save_preset(PROJECT_NAME, preset) - assert [group.name for group in saved.replica_spec_groups] == ["router", "worker"] - assert "cpu=4" in saved.replica_spec_groups[0].resources.pretty_format() - assert "gpu=24GB" in saved.replica_spec_groups[1].resources.pretty_format() - assert len(saved.replica_spec_groups[0].tested_resources) == 1 - assert len(saved.replica_spec_groups[1].tested_resources) == 2 - assert [group.name for group in saved.configuration.replica_groups] == [ + recipe = saved.recipes[0] + assert [group.name for group in recipe.service.replica_groups] == [ "router", "worker", ] + assert "cpu=4" in recipe.service.replica_groups[0].resources.pretty_format() + worker_resources = recipe.service.replica_groups[1].resources + assert worker_resources.gpu is not None + assert worker_resources.gpu.vendor is not None + assert worker_resources.gpu.vendor.value == "nvidia" + assert "gpu=nvidia:24GB" in worker_resources.pretty_format() + assert len(recipe.validations[0].replicas[0].resources) == 1 + assert len(recipe.validations[0].replicas[1].resources) == 2 class TestBuildPresetServiceConfiguration: @@ -627,26 +750,8 @@ def test_get_endpoint_serving_run_name_keeps_long_name_to_avoid_truncation(self) assert get_endpoint_serving_run_name(endpoint_name) == endpoint_name def test_endpoint_name_env_and_profile_are_applied(self): - preset = EndpointPreset( - name="qwen", - model="Qwen/Qwen3-4B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj(_t4_replica_spec_group()) - ], - configuration=ServiceConfiguration.parse_obj( - { - "type": "service", - "name": "preset-name", - "commands": [ - "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", - ], - "port": 8000, - "model": "Qwen/Qwen3-4B", - "env": {"HF_HOME": "/root/.cache/huggingface", "HF_TOKEN": "preset"}, - "resources": {"gpu": "16GB"}, - } - ), - ) + preset = _qwen_preset() + recipe = preset.recipes[0] endpoint_configuration = EndpointConfiguration( name="endpoint-name", model="Qwen/Qwen3-4B", @@ -658,7 +763,7 @@ def test_endpoint_name_env_and_profile_are_applied(self): service_configuration = build_preset_service_configuration( endpoint_name="endpoint-name", endpoint_configuration=endpoint_configuration, - preset=preset, + recipe=recipe, ) assert service_configuration.name == "endpoint-name-serving" @@ -671,23 +776,8 @@ def test_endpoint_name_env_and_profile_are_applied(self): assert service_configuration.resources.gpu is not None def test_builds_repo_less_run_spec(self): - preset = EndpointPreset( - name="qwen", - model="Qwen/Qwen3-4B", - replica_spec_groups=[ - EndpointPresetReplicaSpecGroup.parse_obj(_t4_replica_spec_group()) - ], - configuration=ServiceConfiguration.parse_obj( - { - "type": "service", - "commands": [ - "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", - ], - "port": 8000, - "model": "Qwen/Qwen3-4B", - } - ), - ) + preset = _qwen_preset() + recipe = preset.recipes[0] endpoint_configuration = EndpointConfiguration( name="endpoint-name", model="Qwen/Qwen3-4B", @@ -696,7 +786,7 @@ def test_builds_repo_less_run_spec(self): run_spec = build_preset_run_spec( endpoint_name="endpoint-name", endpoint_configuration=endpoint_configuration, - preset=preset, + recipe=recipe, ) assert run_spec.run_name == "endpoint-name-serving" @@ -745,7 +835,8 @@ async def test_returns_first_preset_with_available_offers( ) assert match is not None - assert match.preset.name == "qwen" + assert match.preset.model == "Qwen/Qwen3-4B" + assert match.recipe.id == "vllm-t4" get_plan_mock.assert_awaited_once() assert get_plan_mock.await_args is not None assert get_plan_mock.await_args.kwargs["run_spec"].run_name == "qwen-endpoint-serving" @@ -780,7 +871,85 @@ async def test_tracks_first_unprovisionable_preset_when_offers_are_unavailable( assert result.provisionable is None assert result.unprovisionable is not None - assert result.unprovisionable.preset.name == "qwen" + assert result.unprovisionable.preset.model == "Qwen/Qwen3-4B" + assert result.unprovisionable.recipe.id == "vllm-t4" + + @pytest.mark.asyncio + async def test_uses_first_later_recipe_with_available_offers( + self, session: AsyncSession, tmp_path + ): + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + _write_qwen_preset_file(presets_dir / "qwen-a.yml", recipe_id="vllm-t4", gpu="16GB") + _write_qwen_preset_file(presets_dir / "qwen-b.yml", recipe_id="vllm-l4", gpu="24GB") + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock( + side_effect=[ + _run_plan_with_offer(available=False), + _run_plan_with_offer(available=True), + ] + ), + ) as get_plan_mock: + result = await find_preset_planning_result( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert result.provisionable is not None + assert result.provisionable.recipe.id == "vllm-l4" + assert result.unprovisionable is not None + assert result.unprovisionable.recipe.id == "vllm-t4" + assert get_plan_mock.await_count == 2 + + @pytest.mark.asyncio + async def test_stops_at_first_recipe_with_available_offers( + self, session: AsyncSession, tmp_path + ): + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + _write_qwen_preset_file(presets_dir / "qwen-a.yml", recipe_id="vllm-t4", gpu="16GB") + _write_qwen_preset_file(presets_dir / "qwen-b.yml", recipe_id="vllm-l4", gpu="24GB") + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(return_value=_run_plan_with_offer(available=True)), + ) as get_plan_mock: + match = await find_matching_preset_plan( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model="Qwen/Qwen3-4B", + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert match is not None + assert match.recipe.id == "vllm-t4" + get_plan_mock.assert_awaited_once() @pytest.mark.asyncio async def test_matching_plan_requires_available_offers(self, session: AsyncSession, tmp_path): @@ -820,23 +989,26 @@ async def test_skips_preset_with_unresolved_env_sentinel( """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - env: - - HF_TOKEN - commands: - - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 - port: 8000 -replica_spec_groups: - - resources: - gpu: 16GB - tested_resources: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 +recipes: + - id: vllm-t4 + service: + env: + - HF_TOKEN + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 """ ) user = await create_user( @@ -903,26 +1075,37 @@ async def test_refreshes_user_ssh_key_before_planning(self, session: AsyncSessio def _write_qwen_preset(tmp_path): presets_dir = _project_presets_dir(tmp_path) presets_dir.mkdir(parents=True) - (presets_dir / "qwen.yml").write_text( + _write_qwen_preset_file(presets_dir / "qwen.yml") + + +def _write_qwen_preset_file( + path, + recipe_id: str = "vllm-t4", + gpu: str = "16GB", +): + path.write_text( """\ type: endpoint-preset model: Qwen/Qwen3-4B -service: - commands: - - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 - port: 8000 -replica_spec_groups: - - resources: - gpu: 16GB - tested_resources: - - cpu: 4 - memory: 16GB - disk: 100GB - gpu: - name: T4 - memory: 16GB - count: 1 -""" +recipes: + - id: {recipe_id} + service: + commands: + - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 + port: 8000 + resources: + gpu: {gpu} + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""".format(recipe_id=recipe_id, gpu=gpu) ) @@ -963,36 +1146,48 @@ def _instance_offer( ) -def _qwen_preset(name: str) -> EndpointPreset: +def _qwen_preset( + recipe_id: str = "vllm-t4", + resources: dict | None = None, +) -> EndpointPreset: + service = ServiceConfiguration.parse_obj( + { + "type": "service", + "name": "preset-service-name", + "commands": [ + "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + ], + "port": 8000, + "model": "Qwen/Qwen3-4B", + "env": { + "HF_HOME": "/root/.cache/huggingface", + "HF_TOKEN": "SECRET_VALUE", + }, + "resources": resources or {"gpu": "16GB"}, + "creation_policy": "reuse", + } + ) return EndpointPreset( - name=name, model="Qwen/Qwen3-4B", - replica_spec_groups=[EndpointPresetReplicaSpecGroup.parse_obj(_t4_replica_spec_group())], - configuration=ServiceConfiguration.parse_obj( - { - "type": "service", - "name": "preset-service-name", - "commands": [ - "vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000", + recipes=[ + EndpointPresetRecipe( + id=recipe_id, + service=service, + validations=[ + EndpointPresetValidation( + replicas=[ + EndpointPresetValidationReplica.parse_obj(_t4_validation_replica()) + ] + ) ], - "port": 8000, - "model": "Qwen/Qwen3-4B", - "env": { - "HF_HOME": "/root/.cache/huggingface", - "HF_TOKEN": "SECRET_VALUE", - }, - "resources": {"gpu": "16GB"}, - "creation_policy": "reuse", - } - ), + ) + ], ) -def _t4_replica_spec_group() -> dict: +def _t4_validation_replica() -> dict: return { - "name": "0", - "resources": {"gpu": "16GB"}, - "tested_resources": [ + "resources": [ { "cpu": 4, "memory": "16GB", From c96456b2df123e9f5d5d386a4dc5599399d03af8 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 8 Jul 2026 15:19:18 +0200 Subject: [PATCH 17/21] Checkpoint endpoint PR cleanup --- .../endpoint-dev-runpod-fleet.dstack.yml | 11 - .../qwen-endpoint-happy.dstack.yml | 10 - .../qwen-endpoint-log-smoke.dstack.yml | 10 - .../no-preset-reapply.dstack.yml | 4 - endpoint-agent-backend-troubleshooting.md | 534 ------ endpoint-agent-checkpoints.md | 752 -------- endpoint-agent-harness-test-plan.md | 784 -------- endpoint-e2e-testing-report.md | 1611 ----------------- endpoint-implementation-plan.md | 921 ---------- .../background-loop.md | 241 --- .../claude-agent-sdk.md | 407 ----- .../config-models.md | 151 -- endpoint-implementation-research/critique.md | 63 - .../dstack-prototyping-skill-audit.md | 115 -- .../dstack-prototyping-skill-outline.md | 11 - .../examples-llm-deployments.md | 129 -- .../external-recipes.md | 85 - .../resource-lifecycle-blueprint.md | 201 -- .../run-submission-internals.md | 126 -- .../service-runtime-health.md | 144 -- .../settings-migrations-testing.md | 127 -- .../skills-and-pr3856.md | 85 - .../verify-findings.md | 147 -- src/dstack/_internal/cli/commands/endpoint.py | 12 +- .../cli/services/configurators/endpoint.py | 10 +- .../_internal/cli/services/endpoint_logs.py | 31 +- .../_internal/cli/commands/test_endpoint.py | 7 +- 27 files changed, 43 insertions(+), 6686 deletions(-) delete mode 100644 .test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml delete mode 100644 .test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml delete mode 100644 .test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml delete mode 100644 .test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml delete mode 100644 endpoint-agent-backend-troubleshooting.md delete mode 100644 endpoint-agent-checkpoints.md delete mode 100644 endpoint-agent-harness-test-plan.md delete mode 100644 endpoint-e2e-testing-report.md delete mode 100644 endpoint-implementation-plan.md delete mode 100644 endpoint-implementation-research/background-loop.md delete mode 100644 endpoint-implementation-research/claude-agent-sdk.md delete mode 100644 endpoint-implementation-research/config-models.md delete mode 100644 endpoint-implementation-research/critique.md delete mode 100644 endpoint-implementation-research/dstack-prototyping-skill-audit.md delete mode 100644 endpoint-implementation-research/dstack-prototyping-skill-outline.md delete mode 100644 endpoint-implementation-research/examples-llm-deployments.md delete mode 100644 endpoint-implementation-research/external-recipes.md delete mode 100644 endpoint-implementation-research/resource-lifecycle-blueprint.md delete mode 100644 endpoint-implementation-research/run-submission-internals.md delete mode 100644 endpoint-implementation-research/service-runtime-health.md delete mode 100644 endpoint-implementation-research/settings-migrations-testing.md delete mode 100644 endpoint-implementation-research/skills-and-pr3856.md delete mode 100644 endpoint-implementation-research/verify-findings.md diff --git a/.test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml b/.test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml deleted file mode 100644 index 31ab9d3e12..0000000000 --- a/.test-configs/endpoint-agent/endpoint-dev-runpod-fleet.dstack.yml +++ /dev/null @@ -1,11 +0,0 @@ -type: fleet -name: endpoint-dev-runpod - -nodes: 0..1 -backends: [runpod] -spot_policy: on-demand -max_price: 0.5 -idle_duration: 5m - -resources: - gpu: 1 diff --git a/.test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml b/.test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml deleted file mode 100644 index 40a1493aa5..0000000000 --- a/.test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml +++ /dev/null @@ -1,10 +0,0 @@ -type: endpoint -name: qwen-endpoint-happy - -model: Qwen/Qwen3-0.6B -preset_policy: reuse-or-create -max_agent_budget: 2 - -spot_policy: on-demand -max_price: 0.5 -backends: [runpod] diff --git a/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml b/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml deleted file mode 100644 index 81e7afbb21..0000000000 --- a/.test-configs/endpoint-agent/qwen-endpoint-log-smoke.dstack.yml +++ /dev/null @@ -1,10 +0,0 @@ -type: endpoint -name: qwen-endpoint-log-smoke - -model: Qwen/Qwen3-0.6B -preset_policy: create -max_agent_budget: 3.0 - -backends: [runpod] -spot_policy: on-demand -max_price: 0.5 diff --git a/.test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml b/.test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml deleted file mode 100644 index 7883ac5c66..0000000000 --- a/.test-configs/endpoint-lifecycle/no-preset-reapply.dstack.yml +++ /dev/null @@ -1,4 +0,0 @@ -type: endpoint -name: endpoint-lifecycle-reapply-smoke -model: dstack/test-endpoint-no-preset -preset_policy: reuse diff --git a/endpoint-agent-backend-troubleshooting.md b/endpoint-agent-backend-troubleshooting.md deleted file mode 100644 index 92815bf9cd..0000000000 --- a/endpoint-agent-backend-troubleshooting.md +++ /dev/null @@ -1,534 +0,0 @@ -# Endpoint Agent Backend Troubleshooting - -This file is for concrete issues found while testing endpoint agent deployments. Keep it -evidence-first: record exact commands, dstack events, native backend observations, and -whether the issue was reproduced independently. - -## Diagnostic Workflow - -Before a live development agent run on a new or suspect hardware path, run an -independent dstack/backend preflight so agent quality is not judged against unknown -capacity: - -1. Check offers with the intended endpoint constraints. -2. For VM-based backends, optionally create/update the fleet separately and wait until - capacity can be pre-provisioned or reused. -3. For container-based backends such as RunPod and Vast.ai, do not treat fleet creation - as pre-provisioning. Use a pinned `nodes: 0..1` fleet template or direct run - constraints, then submit a tiny detached task to force real provisioning. -4. Submit a minimal detached task on the same backend/region/instance class, such as - `nvidia-smi`, with a short `max_duration`. -5. If this preflight fails before image pull/logs, diagnose and record it here as a - dstack/backend issue before running or blaming the endpoint agent. - -When an endpoint agent candidate is stuck or fails, collect evidence in this order. -Prefer the normal dstack control-plane path first; use native provider APIs only when -dstack says an instance exists but the instance cannot be reached or inspected through -the project SSH path. - -1. Endpoint state - - ```bash - uv run dstack endpoint get --json - uv run dstack logs --since 10m - ``` - -2. Backing run state - - ```bash - uv run dstack run get --json - uv run dstack event list --within-run --since 30m - uv run dstack logs --since 30m - ``` - -3. Fleet and instance state - - ```bash - uv run dstack fleet list - uv run dstack event list --within-fleet --since 30m - ``` - -4. Network and SSH reachability - - Check TCP/SSH from the same host where the dstack server runs. Use the project SSH - key that dstack used for the run. Never copy project private keys into the report. - - ```bash - python3 - <<'PY' - import socket, time - host = "" - port = 22 - start = time.time() - try: - s = socket.create_connection((host, port), timeout=8) - print("tcp_connect_ok", host, port, "elapsed", round(time.time() - start, 2)) - s.close() - except Exception as e: - print("tcp_connect_failed", type(e).__name__, str(e), "elapsed", round(time.time() - start, 2)) - PY - ``` - -5. If SSH works, inspect host bootstrap logs - - Start with: - - ```bash - ssh @ 'sudo systemctl status dstack-shim --no-pager || true' - ssh @ 'sudo journalctl -u dstack-shim -n 200 --no-pager || true' - ssh @ 'sudo tail -n 200 /var/log/cloud-init-output.log || true' - ssh @ 'docker ps -a || true' - ssh @ 'nvidia-smi || true' - ``` - -6. Native backend state, only when needed - - Use the existing backend implementation or provider API client only when dstack has - allocated an instance but SSH/shim is unreachable, or when dstack has no hostname and - the provider state is required to understand why. Do not print API keys or credentials. - For CloudRift, use `RiftClient.get_instance_by_id()` and print only sanitized fields - such as status, node mode, host address, VM readiness, and VM state. - -7. Candidate policy - - If a paid candidate remains in backend provisioning with no image pull and no logs for - several polls, stop it and try a different backend or offer. Do not let the agent sit - in a long shell sleep loop. - -## Issue Reports - -### EP-BACKEND-2026-07-04-001: CloudRift VM active in provider API, but dstack run stuck before image pull - -Status: Reproduced with separate minimal tasks. - -Endpoint: - -- Endpoint name: `qwen-endpoint-happy` -- Endpoint id: `3383f4f4-0992-4da2-b9a0-7450a2e9d7b6` -- Agent workspace: `~/.dstack/server/data/endpoint_agent_runs/3383f4f4-0992-4da2-b9a0-7450a2e9d7b6/workspace` - -Candidate run: - -- Run name: `qwen-endpoint-happy-serving` -- Service image: `vllm/vllm-openai:latest` -- Model: `Qwen/Qwen3-0.6B` -- Requested resources: `gpu: 24GB..:1`, `disk: 30GB..` -- Constraints: `spot_policy: on-demand`, `max_price: 0.5` - -Timeline: - -- `2026-07-04 15:29:11` run submitted. -- `2026-07-04 15:29:11` job created. -- `2026-07-04 15:29:11` instance `quick-duck-0` created, status `PENDING`. -- `2026-07-04 15:29:15` job moved `SUBMITTED -> PROVISIONING`. -- `2026-07-04 15:29:15` instance moved to `PROVISIONING`. -- `2026-07-04 15:29:18` run moved `SUBMITTED -> PROVISIONING`. -- No image pull progress appeared. -- No backing service logs appeared. -- `2026-07-04 15:38:14` run was manually stopped to avoid further spend. -- Final run cost reported by dstack: `$0.0592`. - -Actual dstack provisioning data: - -- Backend: `cloudrift` -- Region: `ap-east-tw-kn-1` -- Instance type: `rtx49-10c-kn.1` -- GPU: `RTX4090:24GB:1` -- CPU: `7` -- Memory: `48001 MiB` -- Disk: `1024 GiB` -- Price: `$0.39/hr` -- Host address reported to dstack: `211.21.50.85` -- SSH username: `riftuser` - -Native CloudRift API observation: - -- Instance id: `59aa1b06-77ac-11f1-9b61-937d98c38d58` -- CloudRift status: `Active` -- Node mode: `VirtualMachine` -- Host address: `211.21.50.85` -- VM `ready`: `true` -- VM state: `Running` - -Network observation from the server host: - -```text -tcp_connect_failed timeout timed out elapsed 8.02 -``` - -SSH observation: - -```text -ssh: connect to host 211.21.50.85 port 22: Operation timed out -``` - -What this rules out: - -- This was not a vLLM startup error: the container never reached image pull/startup. -- This was not a model download or dependency error: there were no job logs. -- This was not an endpoint verification failure: the service never became reachable. - -Working hypothesis: - -CloudRift marked the VM active/ready and returned a host address, but the dstack server -host could not reach TCP/22 on that address. The failure is likely in the provider -network path, VM SSH exposure, or early bootstrap path before dstack shim/runner became -reachable. - -Latest independent repro: - -- Repro run: `cloudrift-rtx49-provisioning-smoke` -- Repro time: `2026-07-04 15:58:29` to `2026-07-04 16:00:53` -- Final dstack status: `terminated`, `stopped_by_user` -- Final cost reported by dstack: `$0.015` -- Instance id: `70bb964a-77b0-11f1-bc41-ab4db0e59660` -- Backend/region/instance: `cloudrift`, `ap-east-tw-kn-1`, `rtx49-10c-kn.1` -- Native state while dstack was stuck in provisioning: CloudRift `Active`, - `node_status=Ready`, `node_mode=VirtualMachine`, VM `ready=true`, - VM `state=Running` -- Native host fields: `host_address=211.21.50.85`, - `internal_host_address=10.21.106.99` -- dstack provisioning data after `vm_ready=true`: `hostname=211.21.50.85`, - `ssh_port=22`, `username=riftuser` -- TCP/22 from the dstack server host timed out repeatedly after `vm_ready=true` -- SSH could not be attempted beyond TCP connect because port `22` was unreachable -- `dstack logs cloudrift-rtx49-provisioning-smoke --since 30m` returned no workload - logs -- `image_pull_progress` stayed `null` - -Root-cause status: - -- Reproduced: yes. -- dstack-side behavior diagnosed: yes. -- Provider/backend-side failing boundary: confirmed. CloudRift reported a running, - ready VM and returned a public host address, but the dstack server host could not - reach `host_address:22`. -- Exact provider-side cause is still one level deeper: NAT/firewall/host-address mapping - vs guest SSH/cloud-init. Because SSH never became reachable, host-local logs could not - be inspected from dstack. - -dstack-side diagnosis: - -- `CloudRiftCompute.update_provisioning_data()` treats the VM as provisioned enough to - try SSH once CloudRift returns `virtual_machines[0].ready == true`. -- It then sets `job_provisioning_data.hostname = instance_info["host_address"]` and - leaves `ssh_port = 22`, `username = riftuser`. -- After that, the instance check path attempts an SSH tunnel to the shim through - `riftuser@:22`. -- In this failure, TCP/22 to `211.21.50.85` timed out from the dstack server host, so - dstack could not reach the shim and the run stayed in provisioning until it was - manually stopped. -- This means the endpoint/service/agent code was not the failing layer. The failure - happened between CloudRift reporting VM readiness and dstack being able to establish - SSH to the returned public address. - -Most likely provider-side causes to confirm on a live rerun: - -1. CloudRift returned a public `host_address` before TCP/22 was actually reachable. -2. The CloudRift host address/NAT/firewall did not expose SSH for the VM. -3. Guest `sshd` or cloud-init did not complete even though CloudRift reported VM - `ready: true`. -4. `host_address` was shared or stale and not mapped to the VM's SSH service. - -Evidence for provider/network exposure being the leading hypothesis: - -- Three separate CloudRift runs returned the same public host address `211.21.50.85`. -- The runs had different internal host addresses: - `10.21.106.174`, `10.21.106.75`, and `10.21.106.99`. -- All timed out on TCP/22 from the server host. -- No image pull or job logs were emitted, so the workload never reached container - startup. -- Native CloudRift API later showed the first two instance IDs as `Inactive`, with the - same `host_address`; the latest repro moved to `Deactivating` immediately after stop. - None allowed post-mortem SSH because TCP/22 never became reachable. - -Ready-to-send issue summary: - -```text -Title: CloudRift VM reports ready but dstack server cannot reach returned host_address:22 - -Three dstack runs pinned to CloudRift ap-east-tw-kn-1 / rtx49-10c-kn.1 stayed in provisioning -before image pull/logs. CloudRift API reported VM mode VirtualMachine and VM ready/running, -and dstack received host_address 211.21.50.85 with ssh_port 22 and username riftuser. -From the same host running dstack server, TCP connect to 211.21.50.85:22 timed out and SSH -to riftuser@211.21.50.85 timed out. Two separate minimal nvidia-smi task runs reproduced -the same behavior. Please check whether host_address 211.21.50.85 should expose SSH for these VMs, -whether VM ready can be true before SSH/NAT/firewall is ready, and whether this host address -mapping is stale/shared. - -Affected instance ids: -- 59aa1b06-77ac-11f1-9b61-937d98c38d58 -- d8d2366a-77ad-11f1-9535-cb9f4e143a57 -- 70bb964a-77b0-11f1-bc41-ab4db0e59660 - -Observed host address: 211.21.50.85 -Observed internal host addresses: 10.21.106.174, 10.21.106.75, 10.21.106.99 -``` - -Harness findings: - -- The endpoint agent used a blocking shell wait loop: - - ```bash - until s=$(dstack run get qwen-endpoint-happy-serving --json ...); do - echo "status=$s ...waiting" - sleep 15 - done - ``` - -- This hid intermediate reasoning until the run was externally stopped. -- The agent recorded the planned offer as RunPod A5000 `$0.27`, but the actual - provisioned offer was CloudRift RTX4090 `$0.39`. Learned presets must use actual - `job_provisioning_data`, not the agent's planned candidate note. - -Independent reproduction: - -Run a tiny non-endpoint task constrained to the same CloudRift region and instance type, -then test whether it also gets stuck before image pull. - -Reproduction config: - -```yaml -type: task -name: cloudrift-rtx49-provisioning-smoke - -image: nvidia/cuda:12.4.1-base-ubuntu22.04 -commands: - - nvidia-smi - -resources: - gpu: 24GB..:1 - disk: 30GB.. - -backends: [cloudrift] -regions: [ap-east-tw-kn-1] -instance_types: [rtx49-10c-kn.1] -spot_policy: on-demand -max_price: 0.5 -max_duration: 15m -``` - -Reproduction run: - -- Run name: `cloudrift-rtx49-provisioning-smoke` -- Config file: `cloudrift-rtx49-provisioning-smoke.dstack.yml` -- Submitted: `2026-07-04 15:39:55` -- Manually stopped: `2026-07-04 15:41:28` -- Cost when stopped: `$0.0108` -- Result: Reproduced. - -Reproduction dstack events: - -- `2026-07-04 15:39:55` run submitted. -- `2026-07-04 15:39:55` job created. -- `2026-07-04 15:39:55` instance `quick-duck-0` created, status `PENDING`. -- `2026-07-04 15:39:58` job moved `SUBMITTED -> PROVISIONING`. -- `2026-07-04 15:39:58` instance moved to `PROVISIONING`. -- `2026-07-04 15:40:04` run moved `SUBMITTED -> PROVISIONING`. -- No image pull progress appeared. -- No task logs appeared. - -Reproduction dstack provisioning data: - -- Backend: `cloudrift` -- Region: `ap-east-tw-kn-1` -- Instance id: `d8d2366a-77ad-11f1-9535-cb9f4e143a57` -- Instance type: `rtx49-10c-kn.1` -- GPU: `RTX4090:24GB:1` -- Host address reported to dstack: `211.21.50.85` -- Price: `$0.39/hr` - -Reproduction native CloudRift API observation: - -- CloudRift status: `Active` -- Node status: `Ready` -- Node mode: `VirtualMachine` -- Host address: `211.21.50.85` -- Internal host address: `10.21.106.75` -- VM `ready`: `true` -- VM state: `Running` -- VM name: `ubuntu-jammy-server-gpu-20250904-011801-1783172397` - -Reproduction network observation from the server host: - -```text -tcp_connect_failed timeout timed out elapsed 8.01 -``` - -Reproduction SSH observation: - -```text -ssh: connect to host 211.21.50.85 port 22: Operation timed out -``` - -Reproduction criteria used: - -- Reproduced if the task also stays in `provisioning` with no image pull/logs while - CloudRift reports VM `Active`, `ready: true`, and TCP/22 times out from the server host. -- Not reproduced if the task pulls the image, runs `nvidia-smi`, and exits. -- If SSH becomes reachable, collect `dstack-shim`, cloud-init, Docker, and `nvidia-smi` - logs before stopping/deleting the instance. - -Next harness changes suggested by this incident: - -- For backend provisioning waits, use short status probes and write `agent_state.json` - after each probe. -- After several unchanged provisioning polls with no image pull/logs, inspect dstack - events and native backend state. -- If native backend says the VM is ready but SSH/TCP is unreachable, mark candidate as a - backend provisioning issue and try a different offer/backend. -- Add a candidate result record that distinguishes planned offer from actual - `job_provisioning_data`. - -### EP-HARNESS-2026-07-04-002: Agent recovered with RunPod but failed to return a report - -Status: Reproduced during endpoint happy-path test. - -Context: - -- Endpoint name: `qwen-endpoint-happy` -- First candidate: CloudRift `rtx49-10c-kn.1`, stopped after stuck provisioning. -- Second candidate: RunPod A5000 in `CA-MTL-1`, `$0.27/hr`. - -What happened: - -- After the CloudRift candidate was externally stopped, the agent resumed. -- It grouped offers by backend and chose RunPod A5000 as the next candidate. -- It edited `service.dstack.yml` to constrain the service to RunPod. -- It submitted `qwen-endpoint-happy-serving` again. -- The new RunPod run reached `RUNNING`. -- vLLM started successfully and loaded `Qwen/Qwen3-0.6B`. -- dstack HTTP probes reached `/v1/chat/completions` and returned `200 OK`. -- The endpoint still failed because the server agent process exited before returning a - final verification report. - -RunPod actual provisioning data: - -- Backend: `runpod` -- Region: `CA-MTL-1` -- Instance type: `NVIDIA RTX A5000` -- GPU: `A5000:24GB:1` -- CPU: `9` -- Memory: `51200 MiB` -- Disk: `30 GiB` -- Price: `$0.27/hr` -- Host: `69.30.85.207` -- SSH port: `22198` - -Service evidence: - -- `dstack run get qwen-endpoint-happy-serving --json` reported run status `running`. -- dstack probe success streak reached `5`. -- vLLM logs included: - - ```text - Starting vLLM server on http://0.0.0.0:8000 - Application startup complete. - 127.0.0.1:44128 - "POST /v1/chat/completions HTTP/1.1" 200 OK - ``` - -Direct verification over SSH: - -```text -/v1/models returned model id Qwen/Qwen3-0.6B. -/v1/chat/completions returned HTTP 200 with model Qwen/Qwen3-0.6B. -``` - -Proxy verification issue: - -- Requests through `http://127.0.0.1:3000/proxy/services/main/qwen-endpoint-happy-serving/...` - initially returned `404 Service main/qwen-endpoint-happy-serving not found` before the - job was marked registered. -- After `JobModel.registered` became true, the same proxy path returned `403 - Unauthenticated or unauthorized to access project main` when using the local CLI - project token. -- Direct SSH verification proved the service itself was healthy, so the agent should not - have failed without a final report. - -Harness findings: - -- The agent should not rely on only the public proxy path for final verification. -- If the proxy returns auth/registration errors but dstack probes are passing, the agent - should verify through `dstack attach`/SSH or another dstack-supported direct path. -- The agent must always return a structured final report on terminal success or failure. -- A successful final service should save a preset using actual RunPod provisioning data, - not planned candidate notes. - -### EP-BACKEND-2026-07-04-003: RunPod A5000 offer listed, but pinned smoke task failed with no capacity - -Status: Reproduced with a separate minimal task. - -Context: - -- Purpose: development preflight before judging endpoint agent behavior. -- Backend: `runpod` -- Region: `CA-MTL-1` -- Instance type: `NVIDIA RTX A5000` -- Price shown by offer/apply preview: `$0.27/hr` -- Fleet template: `endpoint-agent-runpod-a5000-template` -- Fleet template nodes: `0..1` - -Important distinction: - -- RunPod is a container-based backend and cannot be pre-provisioned. The fleet with - `nodes: 0..1` is only a pinned template. The actual capacity test is the smoke task. - -Reproduction config: - -```yaml -type: task -name: endpoint-agent-runpod-a5000-smoke - -image: nvidia/cuda:12.4.1-base-ubuntu22.04 -commands: - - nvidia-smi - -resources: - gpu: 24GB..:1 - disk: 30GB.. - -fleets: [endpoint-agent-runpod-a5000-template] -spot_policy: on-demand -max_price: 0.3 -max_duration: 15m -``` - -Offer/apply preview: - -- `dstack offer --backend runpod --gpu 24GB.. --on-demand --max-price 0.5 --region CA-MTL-1` - still listed `NVIDIA RTX A5000` at `$0.27/hr`. -- `dstack apply` preview for the smoke task selected the same offer through the pinned - fleet template. - -Run result: - -- Submitted: `2026-07-04 15:51:13` -- Run id: `6b0a14e5-270e-45bd-8fe2-1e7524a83427` -- Job submission id: `dd5ec06e-442f-4cc3-9601-e7d60ef513ac` -- Status: `failed` -- Job status message: `no offers` -- Termination reason: `failed_to_start_due_to_no_capacity` -- Cost: `$0.0` -- `job_provisioning_data`: `null` -- No image pull progress. -- No logs. - -Events: - -```text -[2026-07-04 15:51:13] [run endpoint-agent-runpod-a5000-smoke] Run submitted. Status: SUBMITTED -[2026-07-04 15:51:13] [job endpoint-agent-runpod-a5000-smoke-0-0] Job created on run submission. Status: SUBMITTED -[2026-07-04 15:51:13] [instance endpoint-agent-runpod-a5000-template-0, job endpoint-agent-runpod-a5000-smoke-0-0] Instance created for job. Instance status: PENDING -[2026-07-04 15:51:14] [job endpoint-agent-runpod-a5000-smoke-0-0] Job status changed SUBMITTED -> TERMINATING. Termination reason: FAILED_TO_START_DUE_TO_NO_CAPACITY -``` - -What this rules out: - -- This was not a model, vLLM, image, or SSH issue: the run failed before provisioning data, - image pull, or logs. -- This did not spend GPU money. - -Working hypothesis: - -The RunPod offer list/apply preview can show a CA-MTL-1 A5000 offer that is not actually -allocatable at submission time. For agent testing, avoid treating a listed offer as proven -capacity until a smoke task has reached provisioning/logs/running. diff --git a/endpoint-agent-checkpoints.md b/endpoint-agent-checkpoints.md deleted file mode 100644 index 40f2c6ff66..0000000000 --- a/endpoint-agent-checkpoints.md +++ /dev/null @@ -1,752 +0,0 @@ -# Endpoint Agent Checkpoints - -This file tracks local endpoint-agent checkpoints: what code version was tested, what -actually worked, how much it cost, and what to improve next. It is intentionally local -and practical, not product documentation. - -## Recovery Workflow - -Use this branch for experimental endpoint-agent work: - -```bash -git switch endpoint-agent-checkpoints -``` - -Each useful checkpoint should have: - -- a git commit -- a local tag named `endpoint-agent/` -- the exact model, endpoint, service run, backend, hardware, and cost -- the focused tests/linters that passed -- links or paths to useful local runtime artifacts -- the main known issues before the next checkpoint - -To return to a known-good checkpoint: - -```bash -git switch endpoint-agent-checkpoints -git checkout endpoint-agent/ -``` - -For normal development after inspecting an old checkpoint: - -```bash -git switch endpoint-agent-checkpoints -``` - -## Checkpoint: qwen-runpod-v0-running - -Status: known-good first real endpoint-agent smoke. - -Expected tag after commit: - -```bash -endpoint-agent/qwen-runpod-v0-running -``` - -Date: 2026-07-04 -Base branch tip before this work: `be48b560e` (`Fix TestGetRunsTable::test_simple_run (#3999)`) - -### What Worked - -- Endpoint `qwen-endpoint-happy` reached `running`. -- Model: `Qwen/Qwen3-0.6B`. -- Agent submitted and verified service run `qwen-happy-v2`. -- Final run ID: `b0831bdc-7456-42ab-8c43-ee250427717f`. -- Endpoint URL: `/proxy/services/main/qwen-happy-v2/v1`. -- Backend/hardware: RunPod `CA-MTL-1`, NVIDIA A40 48GB, 9 CPU, 50GB RAM, 60GB disk. -- Hourly price: `$0.44/hr`. -- Reported run cost at checkpoint time: `$0.0599`. -- Service probe had `success_streak: 20`. -- `dstack logs qwen-endpoint-happy --since 1m` resolved through the endpoint name and showed HTTP 200 model requests. -- Preset was saved at: - `/Users/dstack/.dstack/server/data/endpoint_presets/qwen-qwen3-0-6b-b0831bdc.dstack.yml`. -- Saved preset captured the service YAML plus replica resource evidence for replica group `0`. - -### Verification Commands - -```bash -uv run dstack endpoint get qwen-endpoint-happy --json -uv run dstack run get qwen-happy-v2 --json -uv run dstack logs qwen-endpoint-happy --since 1m -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/cli/commands/logs.py src/tests/_internal/cli/commands/test_logs.py src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -uv run ruff check $(git diff --cached --name-only -- '*.py') -``` - -Results observed on 2026-07-04: - -- focused pytest: `17 passed` -- broader endpoint pytest: `110 passed, 42 skipped` -- focused ruff: `All checks passed!` -- staged Python ruff: `All checks passed!` -- endpoint status: `running` -- service run status: `running` - -### Useful Runtime Artifacts - -These are outside the repo and are not part of the commit: - -- Final report: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/0a17fa5a-da2e-4af7-be12-aeaf6666fc3b/workspace/final_report.json` -- Verification: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/0a17fa5a-da2e-4af7-be12-aeaf6666fc3b/workspace/verification.json` -- Saved preset: - `/Users/dstack/.dstack/server/data/endpoint_presets/qwen-qwen3-0-6b-b0831bdc.dstack.yml` - -### Known Issues / Next Hardening - -- Server logs can still become noisy when agent output contains large YAML/CLI output. -- Endpoint logs now resolve to the backing service, but debug trace storage and readable - endpoint-agent logs need more real-run pressure. -- Agent guidance improved after the first run, but the harness still needs repeated - real examples before we can trust the abstractions. -- CloudRift was excluded from endpoint testing after a separate provisioning failure; - details are in `endpoint-agent-backend-troubleshooting.md`. -- The current smoke is a single-replica vLLM deployment. Later checkpoints need harder - models, different hardware, no-offer fallback behavior, and failed-candidate cleanup. - -### Scope Notes - -This checkpoint should save the code version and the first working evidence. It should -not be treated as v1 quality. The point is to keep a recoverable version while the real -agent loop evolves through repeated tests. - -### Post-Checkpoint Cleanup - -After the checkpoint commit and tag were created, the live endpoint was deleted to stop -GPU spend: - -```bash -uv run dstack endpoint delete qwen-endpoint-happy -y -uv run dstack run get qwen-happy-v2 --json -uv run dstack endpoint get qwen-endpoint-happy --json -``` - -Observed result on 2026-07-04: - -- `dstack endpoint delete qwen-endpoint-happy -y` returned `Endpoint qwen-endpoint-happy deleted`. -- The backing service moved to `terminating`, then `dstack run get qwen-happy-v2 --json` - returned `Run qwen-happy-v2 not found`. -- `dstack endpoint get qwen-endpoint-happy --json` returned `Endpoint not found`. - -### Preset Reuse Preview - -After cleanup, `.test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml` was changed -from `preset_policy: create` to `preset_policy: reuse-or-create` so rerunning the smoke -uses the saved preset first. - -Preview command: - -```bash -echo n | uv run dstack apply -f .test-configs/endpoint-agent/qwen-endpoint-happy.dstack.yml -``` - -Observed result on 2026-07-04: - -- Matched preset: `qwen-qwen3-0-6b-b0831bdc`. -- Matched preset evidence: one verified RunPod A40 replica. -- Planned resources now come from the preset scheduling requirements, not the exact - verified instance resources. -- The command exited at the confirmation prompt; no endpoint or GPU run was created. - -### Agent Status Message Boundary - -Endpoint agent failures now keep endpoint `status_message` compact: agent-originated -errors and failure summaries are collapsed to one line and capped at 500 characters. -Full details remain in the endpoint agent workspace artifacts and endpoint logs. - -Verification on 2026-07-04: - -```bash -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/server/background/pipeline_tasks/endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py -``` - -Observed result: - -- endpoint pytest slice: `112 passed, 44 skipped` -- ruff: `All checks passed!` - -### Endpoint Failure Status UX - -Endpoint tables now show short failure reasons in the `STATUS` column, similar to -`dstack ps`, instead of always showing only `failed`. - -Examples observed locally: - -- `no offers` for an agent-confirmed no-offers/no-capacity endpoint -- `agent failed` for failed agent runs without a verified report -- `no agent` when preset creation requires the server agent but the runtime/key is missing - -The verbose `ERROR` column still contains the endpoint status message. - -Verification on 2026-07-04: - -```bash -uv run pytest src/tests/_internal/cli/utils/test_endpoint.py -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/cli/utils/endpoint.py src/tests/_internal/cli/utils/test_endpoint.py -uv run dstack endpoint -a -``` - -Observed result: - -- endpoint utility pytest: `14 passed` -- broader endpoint pytest: `116 passed, 44 skipped` -- ruff: `All checks passed!` - -### Endpoint Get UX - -`dstack endpoint get NAME` now prints a human-readable endpoint detail table by -default. `--json` remains available for scripts and exact API inspection. - -This makes failed endpoints easier to inspect without requiring verbose list output -or JSON parsing. List/watch output stays compact and continues to show short failure -reasons in `STATUS`. - -Verification on 2026-07-04: - -```bash -uv run pytest src/tests/_internal/cli/utils/test_endpoint.py -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/cli/commands/endpoint.py src/dstack/_internal/cli/utils/endpoint.py src/tests/_internal/cli/utils/test_endpoint.py -uv run dstack endpoint get qwen-endpoint-no-offers-schema -uv run dstack endpoint get qwen-endpoint-no-offers-schema --json -uv run dstack endpoint get --help -``` - -Observed result: - -- endpoint utility pytest: `15 passed` -- broader endpoint pytest: `117 passed, 44 skipped` -- ruff: `All checks passed!` -- plain `endpoint get` shows Project/User/Endpoint/Model/Status/Run/URL/Created/Error -- JSON output and help output still work - -### Endpoint Preset CLI - -`dstack endpoint preset` lists endpoint presets saved on the server. -`dstack endpoint preset get MODEL --json` returns the model-level preset, and -`dstack endpoint preset delete MODEL` removes it after confirmation. - -This is intentionally limited to list/delete. Creating or updating presets remains part -of the endpoint agent flow, where the agent saves a preset only after verifying the final -service. - -Verification on 2026-07-04: - -```bash -uv run pytest src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/core/models/endpoint_presets.py src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/schemas/endpoint_presets.py src/dstack/api/server/_endpoint_presets.py src/dstack/api/server/__init__.py src/dstack/_internal/server/routers/endpoints.py src/dstack/_internal/cli/utils/preset.py src/dstack/_internal/cli/commands/endpoint.py src/dstack/_internal/cli/services/completion.py src/dstack/_internal/cli/main.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/routers/test_endpoints.py -uv run dstack endpoint preset --help -uv run dstack endpoint preset -uv run dstack --help -``` - -Observed result: - -- focused preset/router pytest: `38 passed, 11 skipped` -- broader endpoint/preset pytest: `125 passed, 46 skipped` -- ruff: `All checks passed!` -- `dstack endpoint preset` lists the saved Qwen endpoint presets -- endpoint help includes `preset Manage endpoint presets` - -### Endpoint Preset Resource Contract - -Endpoint presets now separate scheduling requirements from verified runtime evidence: -recipe `service.resources` is used for service planning and offer matching, while -`validations[*].replicas[*].resources` stores exact resources captured from actual -registered service replicas. - -`dstack endpoint preset` now displays one row per recipe, and expands service replica groups with child rows when a recipe has multiple groups. Exact validation resources stay in `get --json`. - -Invalid local preset files are skipped for user-facing preset listing and logged server-side with -the preset path and parse/validation error. - -Verification on 2026-07-04: - -```bash -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/cli/utils/test_preset.py -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/server/services/endpoints/presets.py src/dstack/_internal/server/services/endpoints/preset_building.py src/dstack/_internal/cli/utils/preset.py src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/cli/utils/test_preset.py -uv run dstack endpoint preset -``` - -Observed result: - -- focused preset/endpoint-worker pytest: `69 passed, 35 skipped` -- broader endpoint pytest: `138 passed, 46 skipped` -- ruff: `All checks passed!` -- `dstack endpoint preset` skips the old loose smoke preset and lists the valid learned preset; the server logs the skipped preset path and validation error - -### Endpoint Agent Retest After Deleting Preset - -Retest used the current checkout server on `127.0.0.1:3000` with a temporary CLI -home at `/tmp/dstack-endpoint-test-home`; the normal/default CLI config was restored -to `main -> 127.0.0.1:3002`. - -Flow observed on 2026-07-04: - -- Deleted the existing learned Qwen preset. -- Submitted `qwen-endpoint-smoke` with `backend=runpod`, `spot_policy=on-demand`, - `max_price=0.5`. -- Agent created real service candidates and handled real RunPod capacity failures: - `qwen3-06b-smoke` failed on A5000 no-capacity, `qwen3-06b-smoke2` initially retried - the same no-capacity path, then the agent stopped it and tried `qwen3-06b-l4`. -- `qwen3-06b-l4` first tried L4 and then dstack provisioned A40 in CA-MTL-1 at - `$0.44/hr`; vLLM served `Qwen/Qwen3-0.6B` and real `/v1/chat/completions` - requests returned HTTP 200. -- Endpoint reached `running`; learned preset saved as `qwen-qwen3-0-6b-94071a4a`. -- Cleanup completed: endpoint deleted, `qwen3-06b-l4` stopped. - -Important harness findings: - -- Good: real agent loop works end-to-end through deployment, verification, endpoint - running state, and preset save. -- Bad: the agent copied offer/workaround hardware into final service scheduling - requirements (`gpu.name: [L4, A40, RTX3090]`) instead of preserving the broadest - correct model-derived requirement. -- Bad: the agent's verification/final report said L4, but the actual provisioned - hardware and saved validation resources were A40. The server-side preset builder used - actual run state correctly; the agent report was stale/inferred. - -Patch made after this run: - -- Prompt resources now say to derive scheduling requirements from the model/serving - method, treat preview offers as availability evidence rather than target hardware, - avoid pinning concrete GPU/region/instance unless required or explicitly justified, - and re-read `dstack run get --json` after verification to report actual provisioned - hardware. - -## Checkpoint: qwen-runpod-v1-endpoint-dev-running - -Status: known-good endpoint-agent smoke with separate local project and corrected -preset resource contract. - -Expected local tag after commit: - -```bash -endpoint-agent/qwen-runpod-v1-endpoint-dev-running -``` - -Date: 2026-07-04 -Server: current checkout on `127.0.0.1:3000` -CLI project: `endpoint-dev -> http://127.0.0.1:3000` -Default CLI project preserved: `main -> http://127.0.0.1:3002` - -### What Worked - -- Endpoint `qwen-endpoint-smoke` reached `running`. -- Model: `Qwen/Qwen3-0.6B`. -- Agent submitted and verified service run `qwen-smoke`. -- Final run ID: `cfef76ab-9ec7-4c41-913b-e9595e2979cd`. -- Endpoint URL: `/proxy/services/endpoint-dev/qwen-smoke/v1`. -- Backend/hardware: RunPod `EU-RO-1`, NVIDIA RTX 2000 Ada Generation - (`RTX2000Ada:16GB:1`), 6 CPU, 31GB RAM, 100GB disk. -- Hourly price: `$0.24/hr`. -- Agent service YAML used model-derived scheduling requirements: - `resources.gpu: 16GB..`, not a pinned GPU name, region, or instance type. -- Agent used `vllm serve Qwen/Qwen3-0.6B --port 8000 --max-model-len 8192`. -- dstack service probe reached `success_streak: 4`. -- Agent verified both `/v1/models` and `/v1/chat/completions` through the - dstack service proxy with HTTP 200. -- Endpoint preset was saved as `qwen-qwen3-0-6b-cfef76ab`. -- Saved preset uses broad scheduling requirements and exact tested resources: - - scheduling: `cpu=2.. mem=8GB.. disk=100GB.. gpu=16GB..:1..` - - tested: `cpu=6 mem=31GB disk=100GB gpu=RTX2000Ada:16GB:1` - -### Verification Commands - -```bash -uv run dstack endpoint --project endpoint-dev get qwen-endpoint-smoke --json -uv run dstack run --project endpoint-dev get qwen-smoke --json -uv run dstack endpoint --project endpoint-dev preset -uv run dstack logs --project endpoint-dev qwen-smoke --since 3m -``` - -Observed result on 2026-07-04: - -- endpoint status: `running` -- backing service status: `running` -- preset listed by `dstack endpoint --project endpoint-dev preset` -- `verification.json` recorded HTTP 200 for `/v1/models` and - `/v1/chat/completions` -- `final_report.json` recorded actual provisioned hardware from run JSON - -### Useful Runtime Artifacts - -These are outside the repo and are not part of the checkpoint commit: - -- Final report: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/bb5846b3-4de1-45fb-896f-eaef5ebd73cf/workspace/final_report.json` -- Verification: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/bb5846b3-4de1-45fb-896f-eaef5ebd73cf/workspace/verification.json` -- Saved preset: - `/Users/dstack/.dstack/server/data/endpoint_presets/qwen-qwen3-0-6b-cfef76ab.dstack.yml` - -### Known Issues / Next Hardening - -- After agent verification, the endpoint briefly transitions from `prototyping` to - `provisioning` before `running`. This is confusing; the next patch should avoid - exposing that intermediate status for agent-verified endpoints. -- The agent still sometimes uses invalid CLI forms first, such as `dstack run list` - or uppercase backend names. The harness prompt should make the supported command - surface stricter. -- The agent used a generic run name (`qwen-smoke`). Future prompt/harness rules - should require useful unique names for candidate runs. -- The agent used shell polling loops without explicit timeouts. Future rules should - require bounded polling and clear progress notes. -- `dstack logs -d` can expose very verbose shim environment output. Normal endpoint - logs should become a concise major-event stream written by the agent, while full - trace/debug artifacts stay in the workspace. -- The endpoint agent trace is useful for debugging, but it is too detailed for normal - `dstack logs ENDPOINT`. Endpoint logs should contain major realtime events only: - research/plan summary, candidate submitted, provisioning state, service startup, - verification success/failure, preset save, and cleanup. - -### Post-Checkpoint Patch: Agent Status And Logs - -Implemented immediately after tag `endpoint-agent/qwen-runpod-v1-endpoint-dev-running`. -The tag remains the recovery point for the known-good live RunPod smoke; these edits -are the next local branch commit. - -Status behavior: - -- Agent-created endpoints no longer expose an intermediate `provisioning` state after - the agent returns a verified service run. -- If the reported service is not yet fully visible as a ready dstack service, the - endpoint stays `prototyping` with `service_run_id` linked. -- Once the linked service is ready, the worker saves the learned preset and moves the - endpoint directly from `prototyping` to `running`. -- Preset-based endpoint creation still uses `provisioning`. - -Endpoint log behavior: - -- Claude stream/tool output is no longer copied to endpoint logs. -- Full agent trace and command output remain in workspace artifacts: - `trace.jsonl` when debug is enabled, plus `commands.jsonl` and `command-output/`. -- The agent now has a user-facing progress protocol: append concise JSON objects to - `progress.jsonl`, for example - `{"phase":"submit","message":"Submitted service candidate"}`. -- The server tails `progress.jsonl` and writes only those major events to the configured - log service for `dstack logs ENDPOINT`. -- Endpoint logs still include concise start/finish markers for the provisioning agent. - -Verification on 2026-07-04: - -```bash -uv run pytest src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -uv run pytest src/tests/_internal/cli/commands/test_logs.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints src/tests/_internal/server/services/test_endpoint_presets.py -uv run ruff check src/dstack/_internal/server/background/pipeline_tasks/endpoints.py src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -uv run ruff format --check src/dstack/_internal/server/background/pipeline_tasks/endpoints.py src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -``` - -Observed result: - -- focused endpoint/agent pytest: `51 passed, 36 skipped` -- broader endpoint/log pytest: `104 passed, 11 skipped` -- ruff: `All checks passed!` -- format check: `4 files already formatted` - -## Checkpoint: qwen2-runpod-restart-resource-envelope - -Status: known-good same-host restart plus learned-preset resource-envelope validation. - -Suggested local tag after the next checkpoint commit: - -```bash -endpoint-agent/qwen2-runpod-restart-resource-envelope -``` - -Date: 2026-07-07 -Project: `endpoint-e2e` -Server: current checkout on `127.0.0.1:3000` -Test directory: `/Users/dstack/dstack-endpoints-demo/endpoint-agent-restart` - -### What Worked - -- Endpoint `qwen2-05b-restart-1110` reached `running`. -- Model: `Qwen/Qwen2-0.5B-Instruct`. -- Server restart during `prototyping` reused the same agent session/workspace and did not start a - duplicate Claude process. -- Claude submitted and verified service run `qwen2-05b-restart-1110-1`. -- Final run ID: `c1be5e1a-6c49-4ef5-926e-4d2171a03d98`. -- Backend/hardware: RunPod `EU-CZ-1`, NVIDIA RTX 3090 24GB. -- Hourly price: `$0.46/hr`; final run cost after cleanup: `$0.0484`. -- Claude reported agent cost: `$1.1923`. -- Agent verified `/v1/chat/completions` through the dstack proxy with HTTP 200 and response model - `Qwen/Qwen2-0.5B-Instruct`. -- Endpoint preset was saved as `qwen-qwen2-0-5b-instruct-c1be5e1a`. -- Final service YAML used scheduling resources `gpu: 16GB..24GB:1`, not bare `gpu: 1` and not exact - RTX 3090 resources. -- Saved preset kept reusable scheduling resources separate from exact tested hardware: - - scheduling: `cpu=2.. mem=8GB.. disk=30GB.. gpu=16GB..24GB:1` - - tested: `cpu=32 mem=125GB disk=100GB gpu=RTX3090:24GB:1` -- `dstack endpoint stop qwen2-05b-restart-1110 -y` stopped the endpoint and terminated the backing - service run. -- A no-cost reuse preview for the same model found offers without Claude, but selected the older - duplicate preset `qwen-qwen2-0-5b-instruct-532ddf05`, not this checkpoint's new - `qwen-qwen2-0-5b-instruct-c1be5e1a` preset. - -### Useful Runtime Artifacts - -- Workspace: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace` -- Final report: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace/final_report.json` -- Verification: - `/Users/dstack/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace/verification.json` - -### Known Issues / Next Hardening - -- Claude first wrote `backend` instead of `backends` and omitted `fleets`; the workspace CLI guard - caught both before paid submission, but the prompt/skill should reduce this wasted turn. -- Endpoint logs showed Claude assistant stdout even though `progress.jsonl` was clean. Fixed after - this run: assistant stream text is now trace-only; endpoint logs use `progress.jsonl` and explicit - server lifecycle messages. -- The run recorded successful CUDA/vLLM/FlashAttention behavior but did not capture direct - `nvidia-smi` host driver output. -- Duplicate presets for the same model now exist from repeated tests. Keep that inspectable in v1; - automatic update/repair policy is later, but selection/ranking needs a v1 decision if duplicates - can change which preset is reused. - -### Verification Commands - -```bash -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -uv run pytest src/tests/_internal/core/models/test_endpoints.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/server/routers/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py -uv run ruff check . -uv run pyright -p . -``` - -Observed result after the endpoint-log fix: - -- focused agent pytest: `30 passed` -- broader endpoint pytest slice: `124 passed, 72 skipped` -- ruff: `All checks passed!` -- pyright: `0 errors, 0 warnings, 0 informations` - -## Checkpoint: qwen25-7b-backend-placement-task-first - -Status: known-good backend-placement/task-first validation, with a probe-quality fix applied after -review. - -Suggested local tag after the next checkpoint commit: - -```bash -endpoint-agent/qwen25-7b-backend-placement-task-first -``` - -Date: 2026-07-07 -Project: `endpoint-agent-reasoning` -Endpoint: `qwen25-7b-placement-choice` -Model: `Qwen/Qwen2.5-7B-Instruct` -Fleet: `reusable-vs-container` - -### What Worked - -- The allowed fleet exposed cheaper RunPod container offers and a JarvisLabs L4 reusable/inspectable - offer. -- Claude chose JarvisLabs L4 at `$0.44/hr` over cheaper RunPod offers because it could reuse/inspect - the instance and keep a warm path for the final service. -- Claude submitted a task first: `qwen25-7b-placement-choice-1`. -- Claude then submitted the final service: `qwen25-7b-placement-choice-2`. -- The service reused the warm JarvisLabs instance and verified: - - `/v1/models=200` - - `/v1/chat/completions=200` - - response model matched `Qwen/Qwen2.5-7B-Instruct` -- Endpoint reached `running`, saved a preset, then was stopped cleanly. -- Temporary fleet was deleted afterward; `dstack fleet --project endpoint-agent-reasoning` showed no - fleets. - -### What Was Not Proven - -- The agent did not SSH into the task and did not use a dev environment. -- This proves task-first on reusable backend capacity, not an interactive SSH/dev loop. -- The task probe was too shallow: it observed `nvidia-smi` and driver evidence, but failed on - `python` missing before proving framework import/runtime/server behavior. - -### Fix Applied After Review - -- Endpoint system prompt and `skills/dstack-prototyping/SKILL.md` now say that `nvidia-smi` is host - evidence only. -- A task/dev probe must exercise the intended serving image/runtime/command before it can justify - promotion. -- Final service verification remains the success gate; if final service verification fails, the - agent must return to the evidence loop instead of writing success. - -### Verification Commands - -```bash -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -``` - -Observed result: - -- focused agent pytest: `31 passed` -- broader endpoint/preset/pipeline pytest: `117 passed, 50 skipped` - -## Checkpoint: endpoint-probe-task-shape-guard - -Status: structural harness fix after the batch-task failure. - -Date: 2026-07-07 - -### Why - -The previous prompt/skill change was not enough by itself. In the next live run, Claude added an -optional Hugging Face cache mount, but still encoded the whole probe as a batch task and chose a -RunPod service-like path. That means the agent understood part of the instruction while still -missing the core shape: a probe task should usually be a live environment the agent can attach/SSH -into, not a one-shot shell script. - -### Fix - -- The endpoint agent's local `dstack` wrapper now rejects batch-style endpoint probe tasks. -- Allowed task probe shape: a single long-lived idle command such as `sleep infinity`, with checks - run later through attach/SSH. -- Rejected task probe shape: commands that pack `nvidia-smi`, Python/framework imports, `vllm` / - `sglang`, server startup, or `curl` probes into the task YAML. -- Services are unaffected; the final endpoint proof is still a verified dstack service. - -### Verification Commands - -```bash -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -uv run ruff check src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -``` - -Observed result: - -- focused agent pytest: `33 passed` -- broader endpoint/preset/pipeline pytest: `119 passed, 50 skipped` -- ruff: `All checks passed` - -## Checkpoint: qwen25-probe-quality-cached-batch-abort - -Status: aborted validation run; confirms cache-mount prompt worked but interactive-probe prompt did -not. - -Date: 2026-07-07 -Project: `endpoint-agent-reasoning` -Endpoint: `qwen25-probe-quality-2202` -Fleet: `probe-quality-mixed` - -### What Improved - -- Generated task YAML included an optional Hugging Face instance cache mount: - `/dstack-cache/huggingface` → `/root/.cache/huggingface`, `optional: true`. -- Planned checks still covered vLLM import, local server start, health, and chat completion. - -### What Was Still Wrong - -- The probe remained a batch `commands` chain instead of a long-lived task plus attach/SSH - inspection. -- Claude again wrote `jarvislabs+runpod (container-style)` without stronger backend evidence. -- The actual submitted run landed on RunPod L4, so the placement preference regressed. - -### Cleanup - -- Endpoint `qwen25-probe-quality-2202`: `stopped` -- Task `qwen25-probe-quality-2202-1`: `terminated`, `termination_reason=terminated_by_user` -- Temporary fleet `probe-quality-mixed`: deleted - -### Lesson - -Prompt-only enforcement is not enough here. The next useful change should give the agent explicit -server-generated backend/fleet capability context and should consider a pre-submit guard or required -artifact for interactive probes, instead of adding more generic prose. - -## Checkpoint: qwen25-probe-quality-service-first-abort - -Status: aborted validation run; useful harness failure, not a passing endpoint e2e. - -Date: 2026-07-07 -Project: `endpoint-agent-reasoning` -Endpoint: `qwen25-probe-quality-2136` -Fleet: `probe-quality-mixed` - -### What Happened - -- Temporary fleet exposed RunPod A5000/L4/RTX3090 offers and JarvisLabs L4. -- Claude wrote that both RunPod and JarvisLabs were "container-only (no reusable VM/SSH state)". -- Claude skipped the task/dev probe and submitted direct service `qwen25-probe-quality-2136-1`. -- The service landed on RunPod CA-MTL-1 A5000 at `$0.27/hr`. -- We stopped the endpoint because the run no longer tested the probe-quality fix. - -### Cleanup - -- Endpoint `qwen25-probe-quality-2136`: `stopped` -- Service `qwen25-probe-quality-2136-1`: `terminated`, `termination_reason=terminated_by_user` -- Temporary fleet `probe-quality-mixed`: deleted - -### Lesson - -The failure happened before probe quality. The agent inferred backend capability from the offer -table, turned uncertainty into "no reusable state", and then optimized back to the cheaper RunPod -service-first path. - -Prompt/skill correction after this run: - -- Do not infer "container-only" or "no reusable state" from offers alone. -- Fleet state (`nodes: 0..N`, `idle_duration`, idle/running instances) matters. -- If backend reuse/SSH/cache behavior is uncertain, resolve that uncertainty; do not use it as a - reason to skip a task/dev probe. -- Tiny/well-known model is not enough by itself to skip a probe for a create-recipe endpoint on an - unverified fleet/backend/runtime path. - -## Checkpoint: qwen25-probe-quality-batch-task-abort - -Status: aborted validation run; useful harness failure after backend-choice improvement. - -Date: 2026-07-07 -Project: `endpoint-agent-reasoning` -Endpoint: `qwen25-probe-quality-2152` -Fleet: `probe-quality-mixed` - -### What Improved - -- Claude chose a task probe instead of service-first. -- The task landed on JarvisLabs `L4-1x` at `$0.44/hr`, despite cheaper RunPod offers. -- The planned checks went beyond host visibility: vLLM/Torch/CUDA import, local server start, - `/health`, `/v1/models`, and a local chat completion request. - -### What Was Still Wrong - -- The probe was encoded as one batch `commands` chain instead of a long-lived task plus attach/SSH. -- No instance volumes were configured. Run JSON showed `configuration.volumes=[]`, - `job_spec.volumes=[]`, and `runtime.volume_names=[]`. -- Claude still used imprecise backend wording: `jarvislabs+runpod (container-style)`. - -### Cleanup - -- Endpoint `qwen25-probe-quality-2152`: `stopped` -- Task `qwen25-probe-quality-2152-1`: `terminated`, `termination_reason=terminated_by_user` -- Temporary fleet `probe-quality-mixed`: deleted - -### Fix Applied After Review - -- Endpoint prompt and `skills/dstack-prototyping/SKILL.md` now require long-lived interactive probes - when attach/SSH is available: keep the probe alive with `sleep infinity` or equivalent, attach/SSH - into it, and run bounded checks inside the live environment. -- Batch task commands are now explicitly reserved for unavailable attach/SSH or truly one-shot - checks. -- Prompt/skill now require optional instance cache mounts for Hugging Face-style model caches when - useful, and require the agent to record why cache mounts were omitted. - -### Verification Commands - -```bash -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -``` - -Observed result: - -- focused agent pytest: `31 passed` -- broader endpoint/preset/pipeline pytest: `117 passed, 50 skipped` diff --git a/endpoint-agent-harness-test-plan.md b/endpoint-agent-harness-test-plan.md deleted file mode 100644 index db2327c337..0000000000 --- a/endpoint-agent-harness-test-plan.md +++ /dev/null @@ -1,784 +0,0 @@ -# Endpoint Agent Harness Test Plan - -This document is for testing the agent harness, not for selling the feature. - -The core question is not "can Claude write a service YAML?" The core question is: - -> Can the endpoint agent efficiently converge on a verified dstack service for a model, under real -> project capacity and budget constraints, while leaving enough evidence to improve the harness after -> every failure? - -If a test does not answer that question, it is not a useful harness test. - -## Current Research Baseline - -### Agent runtime facts to rely on - -- Claude Code print/headless mode supports `--output-format stream-json`, `--json-schema`, `--max-turns`, and `--max-budget-usd`. These are useful facts for later runtime governance, but v1 does not expose an endpoint agent-budget field. -- `--max-budget-usd` caps Claude API spend only. It does not cap GPU spend from dstack runs. Budget/cost governance must be designed later with durable per-session accounting before it becomes user-facing. -- Claude Code plugins/skills are useful for packaging context, but they are not the harness. A plugin can expose skills; it does not give us state, candidate accounting, cleanup, resume, spend tracking, or verification gates. -- Public harness engineering writeups emphasize the same lesson: harness design changes outcomes materially, and useful harnesses rely on traces, self-verification/evaluator separation, context handoff artifacts, and controlled execution loops. - -Primary references: - -- https://docs.anthropic.com/en/docs/claude-code/cli-reference -- https://code.claude.com/docs/en/headless -- https://code.claude.com/docs/en/plugins-reference -- https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents -- https://www.anthropic.com/engineering/harness-design-long-running-apps -- https://www.langchain.com/blog/improving-deep-agents-with-harness-engineering -- https://www.langchain.com/blog/the-anatomy-of-an-agent-harness - -### Deployment recipe sources to rely on - -- vLLM has a model recipe index at `https://recipes.vllm.ai/models.json`; as of this check it returned 263 entries and per-model JSON links. -- SGLang exposes `https://docs.sglang.ai/llms.txt`, including cookbook pages for Qwen3, GLM, Llama, and other families, plus advanced pages such as EPD disaggregation. -- `Qwen/Qwen3-0.6B` has a Hugging Face model card with direct vLLM and SGLang launch examples and OpenAI-compatible curl examples. This makes it a good first real model, because the agent should not need to invent the serving path. -- Advanced writeups such as the LMSYS agent-assisted SGLang development post and Wafer GLM-on-AMD post are not v1 requirements, but they define the direction of the harness: model/framework/hardware-specific experimentation, not a generic YAML generator. - -Primary references: - -- https://recipes.vllm.ai/models.json -- https://docs.vllm.ai/ -- https://docs.sglang.ai/llms.txt -- https://docs.sglang.ai/cookbook/autoregressive/Qwen/Qwen3.md -- https://huggingface.co/Qwen/Qwen3-0.6B -- https://qwen.readthedocs.io/en/latest/deployment/vllm.html -- https://qwen.readthedocs.io/en/latest/deployment/sglang.html -- https://www.lmsys.org/blog/2026-07-02-agent-assisted-sglang-development/ -- https://www.wafer.ai/blog/glm52-amd - -### Product / benchmark references to learn from - -These references should influence harness shape and later evaluation, not v1 feature scope. - -- Modal Auto Endpoints frames the product as "one command to a model endpoint" while keeping the generated app/code, GPU selection, regionalization, engine flags, metrics, and benchmark results inspectable. The v1 dstack endpoint agent should copy the inspectability principle, not Modal-specific infrastructure or load tuning. -- Makora frames the frontier as full-stack optimization: orchestration, routing/scheduling, engine tuning, speculative decoding, quantization, kernels, and heterogeneous hardware. This is a useful long-term direction for the agent harness, but most of it is explicitly out of v1. -- Runpod Overdrive's benchmark writeup is useful because it evaluates model serving by workload profile, not just model name. It compares configurations across chatbot, RAG, code-generation, and long-form generation workloads and reports throughput, ITL, TTFT, end-to-end latency, quality, and cost/token. This is Later for endpoint optimization, but v1 artifacts should not make it impossible to add these measurements. - -Additional references: - -- https://modal.com/blog/introducing-auto-endpoints -- https://www.makora.com/ -- https://www.runpod.io/blog/overdrive-benchmarks - -## What We Are Actually Testing - -The harness has five jobs: - -1. Give the agent the right operating context. -2. Bound and observe experiments. -3. Separate candidate execution from final endpoint activation. -4. Preserve evidence for repeatability and improvement. -5. Fail usefully instead of looping or spending blindly. - -The v0 loop should be intentionally simple, but it must already exercise these jobs. - -## Development Preflight: Separate Backend From Agent - -During development, do not use the endpoint agent as the first proof that a backend, -region, instance type, image, SSH path, or fleet/run provisioning path works. Before -asking the agent to spend serious time on a target hardware path, run a small independent -dstack preflight with the same constraints. - -This preflight is a development/testing practice, not a production endpoint requirement. -Normal endpoint usage should still allow the agent and/or preset path to provision -through dstack. - -Recommended preflight order: - -1. Check offers with the same backend/region/instance/max-price/spot constraints. -2. For VM-based backends, optionally create or update a fleet separately and wait until - it can pre-provision or reuse capacity. -3. For container-based backends such as RunPod and Vast.ai, do not treat fleet creation - as pre-provisioning. Use a pinned `nodes: 0..1` fleet template or direct run - constraints, then submit a tiny detached task that forces real provisioning. -4. Submit the tiny task on the target hardware, for example `nvidia-smi` in a CUDA base - image, with a short `max_duration`. -5. Confirm the run reaches image pull/logs/running or records a concrete provisioning - failure. -6. If it fails before image pull/logs, diagnose as a dstack/backend issue first: events, - run JSON, fleet state, native backend state, TCP/SSH, shim/cloud-init where reachable. -7. Record confirmed backend/dstack issues in `endpoint-agent-backend-troubleshooting.md` - with a minimal reproduction. - -Only after this preflight passes should a development run treat agent behavior as the -primary thing under test. This prevents mixing "agent made a poor decision" with "the -selected backend cannot currently provision reachable capacity." - -## Harness Components Under Test - -### Supervisor - -The supervisor is server-side code or a standalone prototype runner that starts the agent process, passes endpoint constraints, streams/traces output, and enforces hard gates. - -Required responsibilities: - -- create isolated workspace and HOME -- create scoped dstack CLI config -- pass only required env vars -- start Claude Code with structured final report schema -- stream compact endpoint logs -- write full debug trace when debug is enabled -- track candidate run names/ids observed from agent output or final report -- stop active candidates on abort/failure when safe -- reject final success without a verified final service report - -### Agent - -The agent is allowed to: - -- read/write files in the workspace -- use web search/fetch -- use shell commands -- invoke the real `dstack` CLI -- create service/task/dev-environment YAMLs for experiments -- submit detached dstack runs within endpoint constraints -- stop bad candidates - -The agent is not allowed to: - -- use hidden server APIs -- bypass endpoint profile constraints -- mark the endpoint running without a real model request -- keep multiple GPU candidates running without an explicit reason -- write secrets to YAML, logs, presets, or final reports - -### Deterministic Observer - -The observer is not an LLM. It reads dstack state, events, logs, and workspace artifacts to decide whether the run satisfied gates. - -Required checks: - -- every submitted run had a recorded candidate entry -- no active non-final candidate remains after terminal failure -- final run exists and is the required service run -- final service has `model` and model URL -- final verification request is recorded -- endpoint constraints were not violated in previews/submissions -- final preset, if saved, includes final service YAML and replica resource evidence -- task/dev probes used for promotion exercised the intended serving stack, not only host/GPU - visibility - -### Placement/Experiment Observer - -For create-recipe e2e runs, the observer must also check that the agent did not -blindly choose the first cheap placement: - -- allowed fleets were used for offer inspection; global offers alone are not enough; -- if a broad fleet exposes multiple backends, the agent compared backend/runtime - characteristics, not just fleet names; -- viable reusable or inspectable placements were identified when present: VM-based, - SSH, Kubernetes, or any backend/runtime path where tasks/dev environments can reuse - image/package/model cache or support interactive diagnosis; -- if such a placement was viable under the endpoint constraints, the first paid - experiment was normally a task or dev-environment style probe, not the final service; -- if the agent chose container-style placement or service-first, `progress.jsonl` contains - a concrete reason, such as no viable reusable offer, - constraint violation, insufficient GPU/disk, much higher price, no useful cache - persistence, or final URL/probe behavior being the only remaining unknown; -- the agent treated hourly price as a constraint, not the objective. A cheaper - container offer is not automatically better than a slightly more expensive reusable - placement if the reusable placement can reduce total iteration time, repeated model - downloads, image churn, or debugging risk; -- the final service was submitted only after the probe removed the main uncertainty, or - after the agent recorded why a probe would not help. -- task-first is not the same as interactive SSH/dev-environment proof. Record whether the - agent had attach/SSH/dev-environment available, whether it used it, and why not if it - skipped it. -- if attach/SSH is available, a probe task should normally stay alive while the agent runs - inspection commands through attach/SSH. A task that packs all checks into one shell - command chain is only acceptable for a genuinely one-shot question or unavailable - attach/SSH. -- a probe is useful only if it reaches the intended recipe evidence: selected image or - install path, Python/framework runtime, model/auth/cache path when feasible, serving - command/port, and ideally a local health or model API request. `nvidia-smi` alone is - host evidence, not service recipe proof. -- for Hugging Face-style model serving, check whether probe and final service YAMLs use - useful optional instance cache mounts, such as Hugging Face and package caches. If not, - the agent must explain why repeated downloads/setup are acceptable for that backend and - model size. -- final service verification remains the success gate. If the final service fails a real - model request, the correct result is another experiment or terminal failure, not a - successful final report based on task/dev evidence. - -This is not a product rule that forbids container backends. It is a harness test: -when reusable/inspectable placement would make the loop faster or more reliable, the -agent should notice and use it. - -Run this as a ladder. First use a cheaper scenario where reusable placement exists -inside a modest budget. Then repeat later with broader/more expensive approved hardware -so the harness is not accidentally tuned only for low-cost small-model cases. - -## Required Workspace Artifacts - -The agent workspace must contain these files by the end of every attempt. They can be written by the agent, the supervisor, or both, but the deterministic observer must be able to parse them. - -### `agent_state.json` - -Purpose: current phase and budget/candidate bookkeeping. - -Required fields: - -```json -{ - "endpoint_name": "qwen-endpoint-agent-smoke", - "model": "Qwen/Qwen3-0.6B", - "phase": "research|capacity|experiment|verify|success|failure", - "max_hourly_price": 0.3, - "started_at": "ISO-8601", - "updated_at": "ISO-8601" -} -``` - -### `sources.jsonl` - -Purpose: recipe and hardware grounding. - -One JSON object per source: - -```json -{ - "url": "https://huggingface.co/Qwen/Qwen3-0.6B", - "kind": "model-card|framework-doc|recipe|dstack-doc|deployment-report|log-evidence", - "claim": "HF card provides vLLM and SGLang launch examples for this model", - "used_for": "serving command selection", - "confidence": "high|medium|low" -} -``` - -### Decision Trail - -Purpose: make deployment decisions reviewable without forcing the agent to write -separate markdown notes. - -Must be visible through `progress.jsonl`, `submissions.jsonl`, `sources.jsonl`, -`verification.json`, and `final_report.json`: - -- model size and serving mode -- expected framework -- expected VRAM and disk class -- selected dstack constraints -- why the selected offer/fleet is credible -- why cheaper/other offers were rejected if applicable - -### `candidates.jsonl` - -Purpose: every spend-capable experiment. - -One JSON object per candidate transition: - -```json -{ - "candidate": "qwen-endpoint-agent-smoke-serving", - "kind": "service|task|dev-environment", - "role": "prototype|final", - "run_id": null, - "status": "planned|previewed|submitted|provisioning|running|verified|rejected|stopping|stopped|failed", - "hourly_price": 0.24, - "resources": "gpu=RTX2000Ada:16GB disk=100GB", - "reason": "first final service candidate from HF vLLM recipe", - "timestamp": "ISO-8601" -} -``` - -### `commands.jsonl` - -Purpose: audit and loop analysis. - -One JSON object per command: - -```json -{ - "timestamp": "ISO-8601", - "command": "printf 'n\\n' | dstack apply -f service.dstack.yml --max-price 0.3 --on-demand", - "exit_code": 0, - "category": "research|offer|preview|submit|status|events|logs|stop|verify", - "output_path": "command-output/0004.txt" -} -``` - -### `verification.json` - -Purpose: activation gate. - -Required on success: - -```json -{ - "run_name": "qwen-endpoint-agent-smoke-serving", - "model_url": "http://...", - "request_kind": "openai-chat-completions|openai-responses|custom", - "request_model": "Qwen/Qwen3-0.6B", - "status_code": 200, - "response_excerpt": "The capital of France is Paris.", - "verified_at": "ISO-8601" -} -``` - -### `final_report.json` - -Purpose: handoff to endpoint worker and preset saving. - -This is the existing structured report, but the harness tests should reject shallow reports that do not reference the artifact evidence. - -Required on success: - -- `success: true` -- final `run_name` or `run_id` -- final `service_yaml` -- `recipe_sources` -- `verification_summary` - -Required on failure: - -- `success: false` -- `failure_summary` -- link to the last useful evidence: candidate, events, logs, source mismatch, or budget/constraint gate - -## Hard Gates - -These gates are deliberately strict because they protect money and prevent false RUNNING endpoints. - -### Before Any Paid Run - -- Endpoint constraints are parsed and rendered. -- Agent has produced at least one source or explicit reason why no external source is needed. -- Hardware reasoning exists. -- `dstack apply` preview was run with the endpoint constraints. -- Preview output does not violate `max_price`, `spot_policy`, backend, region, fleet, instance type, or reuse constraints. -- The user-confirmed hourly budget is still valid. -- For development live tests, the target backend/hardware path passed an independent - dstack preflight, or the test is explicitly about provisioning-stall recovery. - -### While a Candidate Is Running - -- At most one GPU candidate is active unless the supervisor has an explicit multi-candidate allowance. -- If provisioning has no meaningful progress after the configured observation window, inspect events before continuing. -- If there are no service logs, inspect events and run JSON before resubmitting. -- If a run is stopped externally or `termination_reason` is `stopped_by_user`, do not resubmit automatically. -- Do not repeat the same YAML/offer/image after a failure without a recorded new hypothesis. - -### Before Success - -- Final run is the required service run name. -- Final run is a service, not a task or dev environment. -- Final service exposes a model URL. -- Final model request succeeded. -- The request used the requested model name. -- Final YAML is present and does not include secret values. -- Candidate history shows no active leftover GPU runs. - -### On Failure - -- Stop active endpoint-created GPU candidates when safe. -- Do not save a preset. -- Keep workspace artifacts. -- Failure message in endpoint status must be short. -- Detailed evidence must be in endpoint logs/debug trace/workspace, not stuffed into status events. - -## Metrics - -Every real run should end with these numbers: - -| metric | why it matters | -|---|---| -| time to first valid preview | source + config efficiency | -| time to first paid candidate | whether research is stuck | -| time to first logs | infra vs app debugging | -| time to verified model response | actual endpoint value | -| number of paid candidates | spend efficiency | -| total candidate run minutes | GPU spend exposure | -| agent API spend | Claude budget effectiveness | -| repeated command ratio | loop detection | -| repeated failure ratio | harness quality | -| number of source URLs used | grounding | -| number of source URLs that affected final YAML | real grounding vs noise | -| cleanup completeness | leak prevention | - -## Scenario Ladder - -Run these in order. Do not skip ahead just because the previous scenario looked boring. - -### S0: Offline Harness Contract - -Goal: verify artifacts and gates without dstack spend. - -Input: - -```yaml -type: endpoint -name: qwen-endpoint-contract -model: Qwen/Qwen3-0.6B -preset_policy: create -spot_policy: on-demand -max_price: 0.3 -``` - -Run mode: - -- Agent may research and write YAML. -- Agent may run `dstack offer`. -- Agent may run `printf 'n\n' | dstack apply ...`. -- Agent must not run `dstack apply -y`. - -Pass: - -- Required artifacts exist. -- Preview command uses constraints. -- Final report is failure/blocked due no-submit mode, not success. - -Fail: - -- Agent submits a run. -- Agent writes shallow artifacts. -- Agent guesses unsupported flags/YAML. - -### S1: Impossible Constraints - -Goal: no-spend failure on capacity/constraint mismatch. - -Input examples: - -```yaml -type: endpoint -name: qwen-endpoint-no-offers -model: Qwen/Qwen3-0.6B -preset_policy: create -spot_policy: on-demand -max_price: 0.001 -``` - -Pass: - -- No paid run submitted. -- `sources.jsonl`, `progress.jsonl`, and the failure report explain that the model is deployable but constraints block capacity. -- Failure summary is concise. - -Fail: - -- Agent wastes turns trying random YAML. -- Agent suggests violating max price. - -### S2: First Real Happy Path, Tiny Model - -Goal: first actual endpoint created and verified. - -Candidate model: - -- `Qwen/Qwen3-0.6B` - -Why this model: - -- It is small enough for low-cost GPUs. -- Its model card includes direct vLLM and SGLang launch examples. -- The correct path should be easy enough that failures expose harness issues rather than model complexity. - -Initial config: - -```yaml -type: endpoint -name: qwen-endpoint-agent-smoke -model: Qwen/Qwen3-0.6B -preset_policy: create -spot_policy: on-demand -max_price: 0.3 -``` - -Current local capacity observation on 2026-07-04: - -- `dstack offer --max-price 0.3 --on-demand --gpu 1 --max-offers 20` showed multiple offers, mostly Vast.ai plus one RunPod RTX 2000 Ada. -- The previous Vast.ai RTX 5060 Ti attempt stuck in provisioning without service logs. For the next real run, prefer a more stable/common path even if not the cheapest, and ask before increasing `max_price`. -- The CloudRift RTX4090 path reproduced a backend reachability problem with a separate - `nvidia-smi` task; do not use that path for judging agent quality until the backend - issue is understood or resolved. - -Expected agent behavior: - -- Use HF/Qwen source first. -- Prefer vLLM first unless current evidence suggests SGLang is more reliable for the offer/image. -- Use a common CUDA image or Python install path that matches the chosen framework. -- Submit one final service candidate. -- Poll run JSON, events, and logs. -- Verify via OpenAI-compatible chat completion. -- Save preset after success. - -Pass: - -- Verified model response. -- One paid candidate or a clearly justified second candidate. -- Candidate stopped/cleaned if replaced. -- Preset saved with final YAML and observed resources. - -Fail: - -- Agent chooses arbitrary cheapest offer with no reasoning. -- Agent loops on `dstack ps` table parsing. -- Agent resubmits after external stop. -- Agent reports success before model request. - -### S3: Happy Path Replay From Learned Preset - -Goal: prove the preset created by S2 is actually useful without the agent. - -Input: - -```yaml -type: endpoint -name: qwen-endpoint-preset-replay -model: Qwen/Qwen3-0.6B -preset_policy: reuse -spot_policy: on-demand -max_price: 0.3 -``` - -Pass: - -- Plan finds learned preset. -- Plan shows preset and offers. -- Pipeline creates backing service without invoking agent. -- Service reaches running via normal service readiness. - -Fail: - -- Preset lacks enough resource information to replay. -- Preset matching ignores no-offer state. -- Agent is invoked despite `preset_policy: reuse`. - -### S4: Bad Service YAML Recovery - -Goal: test whether the agent can debug its own bad deployment. - -Setup options: - -- Seed prompt/harness with an intentionally bad first hypothesis, such as wrong port or missing `model`. -- Or choose a framework/image combination likely to fail import. - -Pass: - -- Agent notices validation/runtime failure from preview/logs. -- Agent records new hypothesis. -- Agent stops/replaces bad candidate. -- Final candidate is verified. - -Fail: - -- Repeats same apply. -- Treats validation failure as no-offers. -- Leaves failed GPU run active. - -### S5: Provisioning Stall Recovery - -Goal: reproduce the previous stuck-provisioning failure and ensure the agent does not loop. - -Setup: - -- Use a backend/offer class known to have a chance of provisioning stalls, or simulate via fake agent/runtime in tests. - -Pass: - -- Agent polls JSON, then events. -- After bounded no-progress window, agent stops or fails with evidence. -- No resubmission after external stop. - -Fail: - -- Keeps polling table output. -- Dumps huge offer/event output into endpoint status. -- Resubmits after `stopped_by_user`. - -### S6: Env / Gated Model Path - -Goal: verify env handling and secret hygiene. - -Candidate model: - -- A small gated/private HF model if available, or a deliberately missing `HF_TOKEN` scenario. - -Pass: - -- YAML references `HF_TOKEN` by name only. -- Missing/invalid auth is diagnosed from logs. -- Secret value never appears in YAML, endpoint logs, status message, preset, or trace after redaction. - -Fail: - -- Agent writes secret value into YAML. -- Agent treats auth as hardware failure. - -### S7: Medium Model Hardware Reasoning - -Goal: test model/hardware sizing, not just tiny-model happy path. - -Candidate models: - -- `Qwen/Qwen3-4B` -- `Qwen/Qwen3-8B` - -Run only after budget confirmation. - -Expected behavior: - -- Agent estimates VRAM/disk before offers. -- Agent evaluates quantized variants if needed. -- Agent may choose larger GPU over cheapest if reliability/VRAM requires it. - -Pass: - -- Reasoning links model size, precision/quantization, max model length, framework, and selected offer. -- Candidate verifies. - -Fail: - -- Guesses `gpu: 16GB` or `gpu: 80GB` without evidence. -- Confuses parameter size with runtime memory. - -### S8: Framework Choice - -Goal: verify that the harness supports real vLLM vs SGLang choice. - -Input: - -- Use a model with both vLLM and SGLang evidence. - -Pass: - -- Agent records why it chose one framework. -- If first framework fails, switching framework is a new hypothesis with evidence. - -Fail: - -- Agent always uses one framework regardless of source/log evidence. - -### S9: Advanced Scenario, No v1 Implementation - -Goal: ensure advanced sources improve planning without causing premature v1 scope explosion. - -Candidate sources: - -- GLM-on-AMD Wafer post. -- LMSYS agent-assisted SGLang development post. -- SGLang EPD disaggregation docs. - -Pass: - -- Agent identifies advanced deployment patterns and marks them later unless required. -- No multi-service router/worker topology is attempted in v1 by default. - -Fail: - -- Agent tries P/D disaggregation for a simple endpoint. -- Agent ignores advanced evidence for a model that actually needs it. - -## Real Server Path During Development - -Use the actual endpoint worker path for live agent tests. Avoid building a separate -endpoint runner unless there is a specific bug that cannot be isolated through the real -server path, unit tests, workspace artifacts, and the backend preflight above. - -Development live-test order: - -1. Run the independent dstack/backend preflight for the intended hardware path. -2. Apply the endpoint config through `dstack apply -f .dstack.yml`. -3. Watch endpoint logs and the agent workspace artifacts. -4. Stop candidate runs promptly if the agent stalls or the backend preflight evidence no - longer matches reality. -5. If the issue is dstack/backend, reproduce it outside endpoints and report it in - `endpoint-agent-backend-troubleshooting.md`. - -For the next probe-path e2e, success is not just "the endpoint eventually runs." The agent must -show the intended loop: submit a long-lived task/dev probe, attach/SSH into it, run real recipe -checks inside the live environment, then promote a clean service and verify that service through -the model API. A batch task that embeds the whole investigation in `commands` is a harness failure. - -## Server Integration Gates - -Only harden endpoint worker behavior after the real server path or unit tests show it is -needed. Avoid speculative scaffolding. - -### Gate A: Runtime - -- Claude Code subprocess can run non-interactively from server env. -- Budget/cost governance is deferred until it can be enforced durably per endpoint agent session. -- Stream JSON parsing does not block. -- Large tool outputs are kept out of endpoint status. -- Debug trace captures enough detail. - -### Gate B: Candidate Accounting - -- Every paid candidate has a recorded candidate row or artifact. -- Latest/final service run is distinguishable from prototypes. -- Deletion/abort stops active candidates. -- Worker crash recovery can reconcile linked/submitted runs. - -### Gate C: Verification - -- Agent final report alone is not enough. -- Report must reference `verification.json`. -- Server should validate that final run exists and exposes model URL, but the agent owns functional verification. - -### Gate D: Preset Save - -- Preset is saved only after verified success. -- Preset stores final service YAML and ordered replica resource evidence. -- Preset does not include secrets. -- Replay scenario passes. - -## First Real Run Decision - -Next paid run should be S2, but only after confirming: - -- max hourly price -- acceptable backend/provider preference -- whether to avoid Vast.ai for the next attempt after the prior stuck provisioning -- whether to raise max price for a more stable/common NVIDIA offer - -Recommendation: - -- Keep `Qwen/Qwen3-0.6B`. -- Keep `preset_policy: create`. -- Keep `spot_policy: on-demand`. -- Consider preferring RunPod or another stable backend if available under budget, even if Vast.ai is cheaper. -- Do not increase `max_price` without explicit confirmation. - -## What Counts As Harness Improvement - -A change improves the harness only if it moves one of these metrics: - -- fewer paid candidates for same scenario -- lower total GPU minutes -- fewer repeated commands/failures -- better diagnosis on failure -- more reliable cleanup -- better source-to-YAML traceability -- higher replay success from learned preset - -Prompt wording, skills, plugins, or runtime flags are implementation details. They are only useful if the scenario evidence improves. - -## Now vs Later From Product References - -### Now - -- Keep the "one endpoint config to verified service" UX. -- Make the generated service YAML inspectable and save it as preset provenance. -- Record the engine/framework choice and important serving flags in `sources.jsonl`, `verification.json`, and the final report. -- Record enough final-run metadata to reproduce: model, framework, image/install path, command, resources, observed replica resources, and verification request. -- Treat "no hidden black box" as a v1 principle: if the agent made a deployment decision, the workspace artifacts should show why. -- Keep first verification functional: a real model request proves the endpoint works. - -### Later - -- Load testing and workload-profile optimization. -- Benchmarking across chatbot/RAG/code/long-form profiles. -- TTFT, ITL, throughput, end-to-end latency, quality, and cost/token dashboards. -- Autoscaling and multi-replica tuning. -- Speculative decoding, quantization strategy exploration, engine patching, kernel work, and heterogeneous hardware optimization. -- Agent retry policy that uses production traffic metrics to re-tune a running endpoint. -- Curated benchmarked presets or a registry that includes measured workload profiles. - -### Do Not Do In V1 - -- Do not ask the agent to optimize under load. -- Do not gate endpoint RUNNING on benchmark performance. -- Do not add generic performance dashboards before basic deployment/replay works. -- Do not let performance references push us into P/D disaggregation, routers/workers, or custom kernels for simple endpoints. diff --git a/endpoint-e2e-testing-report.md b/endpoint-e2e-testing-report.md deleted file mode 100644 index 1a6e6fbc7d..0000000000 --- a/endpoint-e2e-testing-report.md +++ /dev/null @@ -1,1611 +0,0 @@ -# Endpoint E2E Testing Report - -Date: 2026-07-06 - -## Scope - -This report covers the first real endpoint agent e2e test against a local dstack server with a real RunPod-backed service. The goal was not to prove every endpoint feature, but to validate the most important path: - -1. Apply an `endpoint` configuration. -2. Let the server start the Claude Code based agent. -3. Let the agent use the real `dstack` CLI to create and debug candidate services. -4. Verify the final model endpoint with a real OpenAI-compatible request. -5. Return a final report to the server. -6. Link the endpoint to the final service run. -7. Save a reusable endpoint preset. -8. Mark the endpoint running. - -The test was run outside the repo in `~/dstack-endpoints-demo`. - -## Test Environment - -Project: `endpoint-e2e` - -Endpoint config: - -```yaml -type: endpoint -name: qwen-endpoint-e2e - -model: Qwen/Qwen3-0.6B - -backends: - - runpod -fleets: - - endpoint-e2e-runpod -spot_policy: on-demand -max_price: 0.5 - -preset_policy: create -``` - -Fleet config: - -```yaml -type: fleet -name: endpoint-e2e-runpod - -nodes: 0..1 - -resources: - gpu: 1 - disk: 100GB.. - -backends: - - runpod - -spot_policy: on-demand -max_price: 0.5 -idle_duration: 5m -``` - -Final service run: - -```text -run name: qwen-endpoint-e2e-2 -run id: ea8597ff-4567-418f-9d60-c620991d79a5 -backend: runpod -region: EU-RO-1 -gpu: NVIDIA RTX 2000 Ada Generation, 16GB -price: $0.24/hr -image: vllm/vllm-openai:v0.11.0 -model: Qwen/Qwen3-0.6B -``` - -Final endpoint state: - -```text -qwen-endpoint-e2e running qwen-endpoint-e2e-2 -``` - -Saved preset: - -```text -qwen-qwen3-0-6b-ea8597ff -``` - -Preset path: - -```text -~/.dstack/server/projects/endpoint-e2e/presets/qwen-qwen3-0-6b-ea8597ff.dstack.yml -``` - -## What Worked - -The endpoint apply path successfully created an endpoint record and moved it into the server-side background processing loop. - -The server started the Claude Code agent from the endpoint worker and passed endpoint/project constraints into the agent context. - -The agent used real `dstack` CLI commands, not server helper APIs, to submit candidate services and inspect their state. - -The agent recovered from a failed first serving attempt, selected a different image, submitted a second candidate service, waited for it to run, and verified the model endpoint with a real `/v1/chat/completions` request. - -The final verification was strong enough for v0: HTTP 200, OpenAI-compatible response shape, non-empty generated content, and response model matching `Qwen/Qwen3-0.6B`. - -The server consumed `final_report.json`, found the reported run, verified it was a service owned by the same user/project, linked it through `service_run_id`, saved a preset, and marked the endpoint `running`. - -Endpoint logs now show concise agent progress rather than raw service logs for this run: - -```text -research: Resuming: prior run qwen-endpoint-e2e-1 failed... -candidate: Submitting service qwen-endpoint-e2e-2... -verification: Verified: real chat completion... -done: Service qwen-endpoint-e2e-2 verified... -``` - -The saved preset contains both reusable service requirements and actual tested hardware. For this run, reusable requirements are broad enough for matching: - -```text -cpu=2.. mem=8GB.. disk=100GB gpu=16GB..:1 -``` - -The preset also stores tested hardware separately: - -```text -cpu=6 mem=31GB disk=100GB gpu=RTX2000Ada:16GB:1 -``` - -## What Failed First - -The first candidate run used a bad final service image: - -```text -vllm/vllm-openai:latest -``` - -That resolved to a newer vLLM/CUDA stack requiring a newer NVIDIA driver than the RunPod host exposed. The service failed with a driver/runtime mismatch. This was a useful failure: it proved the agent must not blindly use `latest` for final serving images. - -The initial agent behavior also used a candidate run name too close to the endpoint name. This caused CLI/log ambiguity and made it harder to distinguish endpoint logs from service logs. - -Earlier endpoint logs were too noisy in some cases: large CLI output, service logs, and trace-like content could leak into user-facing endpoint logs. The current run is better, but the logging contract still needs hardening. - -The endpoint stayed in `prototyping` after the service was already healthy because the server intentionally waits for the agent's `final_report.json`. In this run: - -```text -verification.json 10:07:15 -final_report.json 10:07:51 -endpoint running after worker consumed final_report.json -``` - -This is correct behavior, but the UX needs clearer progress messages between "service is healthy" and "endpoint is running". - -## Fixes Made During Testing - -Agent prompt now requires candidate service run names to be distinct from the endpoint name. Suggested naming is short attempt-based names such as: - -```text --1 --2 -``` - -Agent prompt and `dstack-prototyping` skill now warn against unproven `:latest` images for final serving and emphasize image/runtime/driver compatibility. - -The backend value passed into the agent prompt was fixed to use normal lowercase values such as `runpod`. - -The prompt now tells the agent not to wait only on log text. It should poll structured run state and break on terminal statuses. - -The agent flow now asks for normal logs first, using debug logs only when needed. - -Endpoint status naming is now `running` for successful endpoints. The early `active` prototype rows are cleaned directly in the local DB; the runtime no longer carries an `active` alias. - -Endpoint preset storage was moved under the project-scoped server directory: - -```text -~/.dstack/server/projects//presets -``` - -The preset model was adjusted to separate reusable requirements from tested resources. - -Endpoint UX was split from run UX: - -- endpoint progress logs use `dstack endpoint logs `; -- endpoint lifecycle stop uses `dstack endpoint stop `; -- endpoint presets use `dstack endpoint preset ...`; -- top-level `dstack logs` and `dstack stop` remain run-only; -- the old top-level `dstack preset` command was removed. - -Focused agent tests pass: - -```text -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -17 passed -``` - -## Preset Reuse Status - -A reuse preview was run with: - -```yaml -type: endpoint -name: qwen-endpoint-e2e-reuse - -model: Qwen/Qwen3-0.6B - -backends: - - runpod -fleets: - - endpoint-e2e-runpod -spot_policy: on-demand -max_price: 0.5 - -preset_policy: reuse -``` - -The planner did match the saved preset: - -```text -Preset qwen-qwen3-0-6b-ea8597ff -``` - -But it did not find an available offer while the original service was still using the `nodes: 0..1` fleet: - -```text -No matching instance offers available. -``` - -So preset matching is proven at plan level, but actual second deployment from the saved preset is not yet proven. To test it properly, we should either stop the current endpoint/service first or use a fleet that allows another node. - -## Remaining Issues - -The agent harness is still early. It completed one real model deployment, but we should not generalize too much from a small Qwen/vLLM case. - -Endpoint user logs need a stricter contract. They should contain major agent milestones only. Raw command traces, large YAML, huge offer tables, service logs, debug logs, and sensitive config output must stay out of normal endpoint logs. - -The first progress line is still too late. The user should see early progress immediately after the agent starts: what it is checking, what constraints it sees, and whether it is researching, submitting, waiting, or verifying. - -`prototyping` can look stuck when the service is already healthy but the agent is still verifying or writing the final report. We need better progress events, not necessarily another status. - -The agent still needs much better deployment judgment. The `latest` image failure was caught and recovered from, but the next iterations should make image selection, driver compatibility, model memory sizing, and framework choice more systematic. - -The current e2e did not exercise dev-environment based prototyping. That is central to the long-term endpoint idea, especially for harder models, custom recipes, multi-node setups, and performance-sensitive serving. - -The current e2e did not test multi-replica services, replica groups, autoscaling, PD disaggregation, gateways, private models, or larger models. - -The current e2e did not test server restart while Claude is still running. We have code paths for resuming from workspace artifacts, but this needs a real restart test. - -The current e2e did not test multiple server instances with Postgres. The endpoint worker uses the existing lock/pipeline pattern, but the multi-server case still needs an explicit integration test. - -The current e2e did not test budget interruption behavior. Agent budget/cost governance is deferred until dstack can persist and enforce spend per endpoint agent session. - -Debug traces may still contain sensitive command output if the agent runs unsafe inspection commands. This needs a concrete redaction policy before broader testing. - -Cleanup behavior after failed candidate services still needs pressure testing. The agent recovered from `qwen-endpoint-e2e-1`, but cleanup policy for abandoned candidate runs needs to be clear and observable. - -## 2026-07-06 Preset Reuse E2E - -Goal: reapply the stopped `qwen-endpoint-e2e` endpoint with `preset_policy: reuse` from -`~/dstack-endpoints-demo`, prove that the saved preset can create a new service without Claude, and -verify that endpoint lifecycle stays coherent. - -Pre-launch endpoint row: - -```text -id: fa60bac3-bd16-47bf-9c1a-768d5c5025aa -status: stopped -run: - -``` - -Pre-launch run state: - -```text -qwen-endpoint-e2e-2 is still visible in `dstack ps -v`, but it is stopped. -``` - -Observation: keeping the stopped service run in normal run history is consistent with run UX, but -endpoint reuse must not treat it as live ownership. The endpoint row is the source of truth for the -current backing service; old stopped runs are history. - -Launch result: - -```text -Configuration: qwen-endpoint-e2e-reuse-same-name.dstack.yml -Preset: qwen-qwen3-0-6b-ea8597ff -First offer: runpod EU-RO-1, NVIDIA RTX 2000 Ada Generation, $0.24/hr -``` - -Immediate post-launch state: - -```text -endpoint id: fa60bac3-bd16-47bf-9c1a-768d5c5025aa -status: provisioning -service run: qwen-endpoint-e2e-serving -service run id: 6a8f5e0a-cca6-4ae9-942b-aa5625d109ef -preset policy: reuse -``` - -Observation: terminal reapply reused the same endpoint row and reset `created_at` to the new -submission time. This matches the current "one row per endpoint name" direction. The saved preset -path produced a normal service run without invoking Claude. - -Progress at about 50 seconds: - -```text -endpoint: provisioning -run: qwen-endpoint-e2e-serving provisioning -logs: no endpoint log lines -``` - -Observation: empty endpoint logs are fine for preset reuse because no agent is involved. The only -UX gap is apply-time feedback while the CLI is detached: users see `provisioning`, but detailed -progress lives in normal run status/events/logs rather than endpoint logs. - -Progress at about 2-3 minutes: - -```text -instance: runpod EU-RO-1, instance id 4zy2c99vxg9gyl, status BUSY -job: still PROVISIONING -logs: service logs empty -``` - -Observation: offer matching and RunPod instance creation worked. The slow part is after the instance -is available, while the job/service is not yet registered. For real UX, endpoint progress should -distinguish "waiting for capacity", "starting instance", "pulling image", and "waiting for service -probe" when dstack already has those signals. - -Progress at about 3-4 minutes: - -```text -run: running -probe: failing -endpoint: provisioning -logs: first vLLM startup line appears -``` - -Observation: the preset path is not marking the endpoint running just because the service process is -running. It waits for the service/probe signal, which is the right minimum behavior for preset reuse. - -Final preset-reuse state: - -```text -13:42:59 run qwen-endpoint-e2e-serving RUNNING -13:45:24 service replica registered to receive requests -13:45:32 endpoint qwen-endpoint-e2e PROVISIONING -> RUNNING -url: /proxy/services/endpoint-e2e/qwen-endpoint-e2e-serving/v1 -``` - -Result: preset reuse works end to end. It reused the same endpoint row, created a service from the -saved preset without Claude, waited for the service probe/registration signal, and marked the -endpoint `running` with the service proxy URL. - -Observation: the runtime took about five minutes from endpoint apply to endpoint running. Most of -that was normal cold start: RunPod instance provisioning, vLLM image/container startup, model -download, torch compile, CUDA graph capture, and service registration. This is a useful baseline -for future agent/preset comparisons. - -Direct endpoint verification: - -```text -POST /proxy/services/endpoint-e2e/qwen-endpoint-e2e-serving/v1/chat/completions -status: 200 -model: Qwen/Qwen3-0.6B -``` - -Observation: the proxy endpoint is usable after endpoint status becomes `running`. The small Qwen -model did not follow an "exactly reply" instruction cleanly because it emitted reasoning text, so -future verification prompts for this model should check API/model correctness and non-empty content -rather than strict literal instruction following. - -Stop result: - -```text -13:46:42 endpoint marked for stopping -13:46:52 service run RUNNING -> TERMINATING -13:47:06 job TERMINATED and RunPod instance TERMINATED -13:47:18 endpoint STOPPED -``` - -Result: `dstack endpoint stop` stopped the backing service run, terminated the RunPod instance, and -left the endpoint visible as `stopped`. - -Follow-up UX cleanup: stopped endpoints now hide the linked service run in `dstack endpoint` and -`get --json` (`RUN` displays `-`, `run_name` is `null`). The internal `service_run_id` remains in -the database for lifecycle/history purposes, but the user-facing field represents the current live -backing service, not old run history. - -## 2026-07-07 Same-Host Restart And Resource Envelope E2E - -Goal: validate the latest prompt/session changes on a fresh create-policy RunPod endpoint, including -same-host server restart while Claude is running, final service resource envelope quality, final -report handoff, preset save, and stop cleanup. - -Endpoint config: - -```yaml -type: endpoint -name: qwen2-05b-restart-1110 -model: Qwen/Qwen2-0.5B-Instruct - -backends: - - runpod -fleets: - - endpoint-e2e-runpod -spot_policy: on-demand -max_price: 0.5 - -preset_policy: create -``` - -Workspace: - -```text -~/.dstack/server/data/endpoint_agent_runs/6a9ce2c9-2310-4210-9533-0acf47e49d27/1/workspace -``` - -Server restart result: - -```text -endpoint id: 6a9ce2c9-2310-4210-9533-0acf47e49d27 -Claude pid before restart: 83844 -Claude pid after restart: 83844 -duplicate Claude process: no -endpoint state: prototyping -session/workspace: reused -``` - -This validates the same-host case: restarting the server while Claude was already running did not -create a second agent process or a second agent session. - -Final service run: - -```text -run name: qwen2-05b-restart-1110-1 -run id: c1be5e1a-6c49-4ef5-926e-4d2171a03d98 -backend: runpod -region: EU-CZ-1 -gpu: NVIDIA GeForce RTX 3090, 24GB -price: $0.46/hr -image: vllm/vllm-openai:v0.24.0 -model: Qwen/Qwen2-0.5B-Instruct -``` - -The final service YAML used the intended scheduling envelope instead of copying exact hardware: - -```yaml -resources: - gpu: 16GB..24GB:1 - disk: 30GB.. -``` - -The saved preset preserved that distinction: - -```text -preset: qwen-qwen2-0-5b-instruct-c1be5e1a -scheduling: cpu=2.. mem=8GB.. disk=30GB.. gpu=16GB..24GB:1 -validation: cpu=32 mem=125GB disk=100GB gpu=RTX3090:24GB:1 -``` - -This is the important preset-quality result from the run: reuse planning can stay broad enough to -find equivalent 16-24GB offers, while exact verified placement remains inspectable through -`dstack endpoint preset get --json`. - -Verification and handoff: - -```text -service running: 09:22 UTC -model API HTTP 200: 09:22:46 UTC -final_report.json: 09:23:24 UTC -endpoint running: 09:23:52 UTC -endpoint URL: /proxy/services/endpoint-e2e/qwen2-05b-restart-1110-1/v1 -``` - -The endpoint correctly stayed `prototyping` after the service was healthy until the agent produced a -structured verification report. Once Claude exited, the worker consumed the report, linked the -service run, saved the preset, and moved the endpoint to `running`. - -Stop cleanup: - -```text -endpoint stop requested: 09:24 UTC -service run status: terminated -termination reason: stopped_by_user -endpoint status: stopped -final run cost: $0.0484 -Claude reported cost: $1.1923 -``` - -The stop path terminated the RunPod service and left the endpoint visible as `stopped`. - -Remaining issues from this trace: - -- Claude first wrote `backend: [runpod]` instead of `backends: [runpod]`, and initially omitted - `fleets`; the workspace CLI guard caught both before any paid submission and Claude corrected the - YAML. The guard worked, but the prompt/skill should still reduce this waste. -- The endpoint log stream still included Claude assistant stdout such as "Let me..." and the final - markdown summary, even though `progress.jsonl` itself was clean. Fixed after the run: endpoint logs - now come from `progress.jsonl` and explicit server lifecycle messages only; Claude stdout remains - in trace/artifacts. -- The agent treated early proxy `404 Service not found` as a retryable registration/startup delay and - then verified successfully. That behavior was good, but the prompt should keep this tied to run JSON - and service readiness evidence, not blind retries. -- The report recorded that CUDA/FlashAttention/NCCL initialized successfully, but still did not - capture a direct `nvidia-smi` host driver string. For presets, this should remain explicit: - runtime worked; exact host driver was not recorded. -- Another preset for the same model already existed from earlier testing. Duplicate learned recipes - are acceptable for v1 inspection; v1 selection is intentionally simple and deterministic. - -Reuse preview after cleanup: - -```text -configuration: qwen2-05b-reuse-1110.dstack.yml -preset_policy: reuse -selected preset: qwen-qwen2-0-5b-instruct-532ddf05 -offers: 7 matching RunPod offers, first RTX 2000 Ada at $0.24/hr -submitted: no -``` - -This proved that a Qwen2 preset is reusable without Claude, but it did **not** prove reuse of the -new `c1be5e1a` preset. The planner selected the older duplicate recipe, whose scheduling resources -are looser (`gpu: 1`, `disk: 100GB`) than the new resource-envelope recipe. That is now accepted v1 -behavior: try stored recipes in order and use the first recipe whose normal run plan has available -offers. Automatic ranking/update remains deferred. - -## 2026-07-07 Qwen2.5 Preset Reuse And CLI UX Check - -Goal: validate the preset-reuse path for the model-level recipe format after the latest CLI/storage -changes, and inspect whether the endpoint/preset tables are understandable during normal use. - -Test directory: - -```text -~/dstack-endpoints-demo/e2e-2026-07-07 -``` - -Important testing note: running `uv run dstack` from outside the repo can pick up the installed -package version instead of this branch. Use the branch binary for isolated e2e runs: - -```text -/Users/dstack/dstack/.venv/bin/dstack ... -``` - -Endpoint config: - -```yaml -type: endpoint -name: qwen25-05b-reuse-check -model: Qwen/Qwen2.5-0.5B-Instruct - -preset_policy: reuse -fleets: - - endpoint-e2e-runpod -backends: - - runpod -spot_policy: on-demand -max_price: 0.5 -``` - -Plan/preview: - -```text -preset: Qwen/Qwen2.5-0.5B-Instruct -recipe: 4a73893b -offers: 5 matching RunPod offers under the endpoint constraints -``` - -Paid run: - -```text -endpoint submitted: 2026-07-07 10:33:46 UTC -service run: qwen25-05b-reuse-check-serving -actual placement: runpod CA-MTL-1, NVIDIA RTX A5000 24GB -price: $0.27/hr -``` - -The service emitted vLLM startup logs, downloaded and loaded the model, compiled/warmed up, exposed -the OpenAI-compatible API, and served real chat-completion requests. The endpoint stayed -`provisioning` until the existing service probe/registration path passed, then moved to `running`. - -Direct endpoint verification: - -```text -POST /proxy/services/endpoint-e2e/qwen25-05b-reuse-check-serving/v1/chat/completions -status: 200 -model: Qwen/Qwen2.5-0.5B-Instruct -reply: non-empty assistant message -``` - -Stop result: - -```text -endpoint status: stopped -service run: terminated/stopped -active spend: no live GPU run left -``` - -What this proves: - -- The no-Claude preset-reuse path still works after moving presets to model-level recipes. -- The server waits for real service readiness, not only process startup. -- A learned recipe can plan on one currently available offer and land on another valid offer as - provider availability changes; the actual placement belongs in run/provisioning history and - future validation evidence, not in the apply table as a promise. -- Empty endpoint logs for `preset_policy: reuse` are acceptable for now because no agent is running. - If users need richer preset-path progress later, it should come from server lifecycle events, not - service stdout. - -CLI/storage fixes made after this check: - -- `dstack endpoint` now hides the backing service run in the default table and shows `POLICY`. - The service run remains visible in verbose/detail/JSON output as debugging information. -- Default endpoint listing now behaves like `ps`: show unfinished endpoints; if none exist, show the - latest finished endpoint. -- `dstack endpoint preset` now groups rows by model and shows only the scheduling GPU by default. - `-v` shows full service scheduling resources. Child `recipe=0`, `recipe=1`, ... rows appear - only when a model has multiple recipes. Validation counts were removed from the compact table; - exact validation evidence remains in `dstack endpoint preset get --json`. -- The agent harness now asks Claude to make final GPU service resources vendor-aware when the - selected/proven hardware is vendor-specific, e.g. `gpu: nvidia:16GB..24GB:1`. The preset loader - preserves existing vendorless local recipes instead of rewriting them on read. -- The local preset store now merges duplicate files by model for list/get/planning, and save/delete - collapse or remove all files for that model. In the live project, - `dstack endpoint preset get 'Qwen/Qwen2-0.5B-Instruct' --json` now returns both recipe ids - (`c04afca5`, `79a6b0b7`) consistently with the list output. - -Verification after code changes: - -```text -endpoint-focused pytest: 185 passed, 72 skipped -focused preset/CLI pytest: 57 passed -ruff: All checks passed -pyright: 0 errors -live endpoint table: default shows NAME/MODEL/STATUS/POLICY/CREATED -live preset table: default shows MODEL/GPU, grouped by model with recipe ordinals only when needed -``` - -## 2026-07-07 Broad-Fleet Vendor-Aware Harness Check - -Goal: remove the RunPod-only test bias, verify that the endpoint does not force a backend, and prove -that new learned recipes can be vendor-aware without mutating older local vendorless recipes. - -Test fleet: - -```yaml -type: fleet -name: endpoint-e2e-gpu -nodes: 0..1 -resources: - gpu: 1 - disk: 30GB.. -backends: - - lambda - - verda - - jarvislabs - - runpod -spot_policy: auto -max_price: 0.5 -idle_duration: 5m -``` - -Important observation: the fleet was no longer RunPod-only, but the currently available matching -offers under `$0.5/hr` were still all RunPod. Lambda, Verda, JarvisLabs, and Vast.ai returned no -matching GPU offers in this local project even with relaxed price checks. So the agent selecting -RunPod in this test was explained by current capacity/offer reality, not by endpoint config. - -Endpoint config: - -```yaml -type: endpoint -name: qwen25-broad-fleet -model: Qwen/Qwen2.5-0.5B-Instruct -preset_policy: create -fleets: - - endpoint-e2e-gpu -spot_policy: auto -max_price: 0.5 -``` - -Result: - -```text -service run: qwen25-broad-fleet-1 -run id: b601b053-0b65-4b88-a1ba-53a4ecd15423 -actual placement: runpod EU-RO-1, NVIDIA RTX 2000 Ada 16GB -price: $0.24/hr -endpoint state: running, then stopped -``` - -The agent wrote a clean final service YAML: - -```yaml -fleets: - - endpoint-e2e-gpu -max_price: 0.5 -spot_policy: auto -resources: - gpu: nvidia:16GB..24GB:1 - disk: 30GB.. -``` - -The agent verified the model with a real OpenAI-compatible chat request. The server consumed -`final_report.json`, linked the endpoint to the final service run, saved a new vendor-aware recipe, -and marked the endpoint `running`. - -Reuse preview after stopping the live endpoint: - -```text -configuration: qwen25-broad-reuse-preview.dstack.yml -preset_policy: reuse -selected model preset: Qwen/Qwen2.5-0.5B-Instruct -selected recipe: 4a73893b -offers: 10 matching RunPod offers, first RTX 2000 Ada at $0.24/hr -submitted: no -``` - -The preview initially showed no offers while the only `nodes: 0..1` fleet slot was occupied by the -live endpoint. After stopping the endpoint, the same preview showed offers. That behavior is -conventional capacity accounting, not a preset matching bug. - -Issues found: - -- Endpoint logs were blank for the first ~minute while Claude inspected the workspace/fleet. Normal - endpoint logs need earlier useful progress, even though debug traces already captured commands. -- Older local same-model recipes remain selectable before newer vendor-aware recipes if they are - provisionable. That is now the explicit v1 rule: try stored recipes in order and use the first - recipe whose normal run plan has available offers. Explicit `recipe: ` selection is deferred. -- The agent still wrote a slightly stale final image (`vllm/vllm-openai:v0.6.6`) for a tiny model. - It worked, but image choice needs stronger grounding against current vLLM/SGLang guidance in - harder scenarios. - -Follow-up log-smoke fix: - -```text -endpoint: qwen25-log-start-smoke -purpose: create-policy smoke, stopped before any service submission -result: endpoint logs showed the server startup progress line and Claude's first progress line -service: no new service run submitted -``` - -Normal endpoint logs now get an immediate server-written line when the detached Claude session starts: - -```text -Starting endpoint prototyping agent for Qwen/Qwen2.5-0.5B-Instruct. Allowed fleets: endpoint-e2e-gpu. The agent will inspect offers, choose a service recipe, deploy it, and verify the model API before the endpoint becomes running. -``` - -The prompt now also requires Claude's first workspace action to be a short `progress.jsonl` message -before inspecting state, fleets, offers, recipes, logs, or model docs. - -## Assessment - -This was the first meaningful proof that the endpoint idea can work end to end: endpoint config in, real agent work, real service deployed, real model request verified, endpoint running, preset saved. - -The biggest result is not that Qwen 0.6B runs. The important result is that the system survived a real failed candidate, recovered through agent reasoning, and produced a verified final service that the server could link and persist. - -The biggest remaining risk is still the harness quality. For v1, the server plumbing is useful only if the agent loop becomes strong at model/framework/hardware reasoning, uses dstack efficiently, keeps logs readable, and leaves enough trace evidence to improve each failed run. - -## Next Tests - -1. Reduce the YAML/property waste exposed by the guard (`backend` vs `backends`, omitted `fleets`) - without weakening the guard. -2. Run existing-fleet scenarios that expose both reusable/inspectable backend placement - (VM-based, SSH, or Kubernetes) and container-style placement if possible. Start with a cheaper - scenario, then test broader/more expensive hardware separately after budget approval. Each test - should verify that the agent compares backend/runtime characteristics inside the allowed fleets, - treats hourly price as a constraint rather than the objective, prefers the reusable placement when - viable, starts with a task/dev-style probe when that removes meaningful uncertainty, and records a - concrete reason if it chooses container placement or service-first. -3. Test private/Hugging Face gated model env handling. -4. Run a larger common model after confirming budget and target hardware. -5. Test multiple server instances with Postgres; same-host restart is now tested, multi-host is not. -6. Add log/trace redaction tests and keep endpoint logs limited to progress-level messages. -7. Add regression tests for final-report-to-running handoff, preset reuse, and cleanup of non-final - submitted runs after restart. - -## 2026-07-07 no-cost recipe-selection check - -Regression tests now cover the v1 selection rule directly: - -- if the first stored recipe has no available offers and the next recipe does, the second recipe is - selected and the first is retained as the first unprovisionable match; -- if the first stored recipe has available offers, planning stops there and does not inspect later - recipes. - -Live no-cost preview from `~/dstack-endpoints-demo/e2e-2026-07-07`: - -```text -configuration: qwen25-broad-reuse-preview.dstack.yml -project: endpoint-e2e -preset_policy: reuse -selected model preset: Qwen/Qwen2.5-0.5B-Instruct -selected recipe: 4a73893b -offers: 10 matching offers, first RunPod RTX 2000 Ada at $0.24/hr -submitted: no -``` - -The preview was run with `n` at the confirmation prompt, so it did not create an endpoint and did -not invoke Claude. Running the same preview by absolute path from the repo still hits the existing -generic CLI `relative_to(Path.cwd())` path error for configs outside the current directory; that was -observed but intentionally left untouched here. - -## 2026-07-07 prompt/guard hardening for YAML constraints - -Live traces repeatedly showed Claude confusing service YAML fields with CLI flags, especially -writing singular `backend` or omitting `fleets` and relying on the guard to catch it. The guard -stays as the hard safety net, but the prompt/skill should reduce those failed previews. - -Changes made: - -- endpoint prompt now explicitly separates plural service YAML fields (`fleets`, `backends`, - `regions`, `instance_types`) from singular CLI flags (`--fleet`, `--backend`, `--region`, - `--instance-type`); -- `dstack-prototyping` now repeats that distinction near the fleet-filtered offer guidance and the - final service promotion checklist; -- workspace CLI guard now returns targeted errors for singular run-config keys such as `fleet` and - `backend`, before falling through to generic missing-constraint messages. - -No paid run was needed for this step. Tests covered prompt inclusion and guard behavior. - -## 2026-07-07 no-cost capacity preflight - -Current project/fleet state: - -```text -project: endpoint-e2e -fleet: endpoint-e2e-gpu -shape: nodes 0..1, gpu:1, disk 30GB.., spot auto, max_price $0.5 -backends: lambda, verda, jarvislabs, runpod -live instances: none -active endpoints: none -``` - -The last visible run row was only history: - -```text -qwen25-broad-fleet-1: terminated, stopped_by_user, price $0.24/hr -``` - -Offer checks: - -- `endpoint-e2e-gpu` under `$0.5/hr`: RunPod only. -- `endpoint-e2e-gpu` under `$1/hr`: still RunPod only. -- Lambda, Verda, and JarvisLabs through this fleet: no matching offers. -- Nebius is not configured in `~/.dstack/server/config.yml`. - -Conclusion: the current project cannot test the desired VM/SSH-capable prototyping path. A paid -agent e2e using the current fleet will most likely be a RunPod/container-backed test again. That is -still useful for checking the prompt/guard fix, but it will not validate dev-environment-style -reuse on a VM fleet. - -## 2026-07-07 backend-placement e2e setup - -Goal: create a no-spend setup that can actually test whether the agent prefers reusable/inspectable -backend placement and task-first probing when that makes sense. The important distinction is that -fleets define allowed capacity, while backend placement determines the prototyping strategy. - -Test directory: - -```text -~/dstack-endpoints-demo/backend-placement-2026-07-07 -``` - -Project: - -```text -endpoint-agent-reasoning -``` - -Fleet template applied with `nodes: 0..1`, so no GPU spend was started: - -```yaml -type: fleet -name: reusable-vs-container - -nodes: 0..1 - -resources: - gpu: nvidia:24GB:1 - disk: 100GB.. - -backends: - - jarvislabs - - runpod - -spot_policy: auto -max_price: 1.5 -idle_duration: 30m -``` - -Fleet-filtered offers under `$1.5/hr`: - -```text -runpod RTX A5000 24GB $0.27/hr -runpod RTX A5000 24GB $0.27/hr -runpod L4 24GB $0.39/hr -jarvislabs L4 24GB $0.44/hr -runpod RTX 3090 24GB $0.46/hr -runpod RTX PRO 4000 $0.57/hr -``` - -This is a useful harness test shape: cheaper container-style placements exist, but there is also a -viable JarvisLabs L4 placement inside the same allowed fleet. The next paid endpoint run should -validate whether the agent compares backend/runtime characteristics, treats hourly price as a -constraint rather than the objective, prefers the reusable/inspectable placement when it improves the -total experiment loop, starts with a task/dev-style probe before the final service, and records a -concrete reason if it chooses RunPod or service-first. - -Prepared endpoint config: - -```yaml -type: endpoint -name: qwen25-7b-placement-choice -model: Qwen/Qwen2.5-7B-Instruct - -preset_policy: create -fleets: - - reusable-vs-container -spot_policy: auto -max_price: 1.5 -``` - -No-spend endpoint preview showed the create-policy agent path and exited at confirmation. The paid -run result is below. - -## 2026-07-07 backend-placement / task-first e2e - -Endpoint: - -```text -project: endpoint-agent-reasoning -endpoint: qwen25-7b-placement-choice -model: Qwen/Qwen2.5-7B-Instruct -fleet: reusable-vs-container -``` - -Allowed capacity intentionally exposed both cheaper container placement and a reusable/inspectable -VM-style placement: - -```text -runpod RTX A5000 24GB $0.27/hr -runpod L4 24GB $0.39/hr -jarvislabs L4 24GB $0.44/hr -runpod RTX 3090 24GB $0.46/hr -``` - -Observed agent route: - -- Chose JarvisLabs L4 over cheaper RunPod offers because JarvisLabs was reusable/inspectable and - could keep a warm instance inside the same allowed fleet. -- Submitted a task first: `qwen25-7b-placement-choice-1`. -- Then submitted the final service: `qwen25-7b-placement-choice-2`. -- The service reused the warm JarvisLabs L4 instance and reached `running`. -- Final verification succeeded through the dstack service URL: `/v1/models=200` and - `/v1/chat/completions=200` for `Qwen/Qwen2.5-7B-Instruct`. -- The endpoint moved to `running`, saved the learned recipe, then was stopped cleanly. -- The temporary fleet was deleted; a later fleet listing for `endpoint-agent-reasoning` showed no - active fleets. - -Final service evidence: - -```text -run id: dbc160ff-8e44-4beb-bdcb-7a294c1a5d44 -backend: jarvislabs -region: india-noida-01 -instance: L4-1x -gpu: NVIDIA L4 24GB -driver: 580.126.20 -CUDA reported: 13.0 -price: $0.44/hr -image: vllm/vllm-openai:v0.11.0 -``` - -What this proves: - -- The agent can notice a reusable/inspectable backend when a cheaper container backend is also - available inside the same allowed fleet. -- The agent can choose task-first before the final service. -- A warm reusable instance can reduce the final service placement loop. -- The server correctly waited for the agent's final service verification report before marking the - endpoint `running`. - -What this does not prove: - -- It did not use SSH or a dev environment. This proves task-first on a reusable backend, not an - interactive SSH/dev loop. -- The task probe was too shallow. It successfully observed `nvidia-smi`, host driver, and GPU - memory, but then failed on `/bin/sh: 1: python: not found`. The agent over-interpreted that as - full image/runtime compatibility. The final service still proved the endpoint, but the probe did - not prove framework import, `torch.cuda`, model download, local server start, or local API shape. - -Required harness correction from this run: - -- A task/dev probe must exercise the intended serving image/runtime/command, not only the host. -- `nvidia-smi` is host evidence, not service recipe evidence. -- If a probe exits before framework/runtime/server checks, it is failed or inconclusive evidence. -- The clean service remains the final authority. If final service verification fails, the agent must - go back to task/dev or submit a changed service; it must not write success. - -## 2026-07-07 probe-quality rerun aborted after service-first decision - -Goal: validate the probe-quality prompt fix with a fresh create-policy endpoint. - -Project and setup: - -```text -project: endpoint-agent-reasoning -endpoint: qwen25-probe-quality-2136 -model: Qwen/Qwen2.5-0.5B-Instruct -fleet: probe-quality-mixed -``` - -The temporary fleet intentionally exposed the same placement tension as the previous validation: - -```text -runpod A5000 24GB $0.27/hr -runpod L4 24GB $0.39/hr -jarvislabs L4 24GB $0.44/hr -runpod RTX3090 $0.46/hr -``` - -What happened: - -- Claude saw JarvisLabs and RunPod in the fleet-filtered offers. -- Claude wrote that both backends were "container-only (no reusable VM/SSH state)". -- Based on that claim, it skipped the task/dev probe. -- It selected the cheaper RunPod A5000 and submitted service `qwen25-probe-quality-2136-1`. -- We stopped the endpoint before waiting for service verification because the run no longer tested - the intended probe-quality fix. - -Cleanup: - -```text -endpoint: qwen25-probe-quality-2136 stopped -service: qwen25-probe-quality-2136-1 terminated_by_user -backend: runpod CA-MTL-1 A5000, $0.27/hr -fleet: probe-quality-mixed deleted -``` - -Why this matters: - -- The issue was not that Claude failed to run a deep enough task probe; it did not run a probe at - all. -- The deeper issue is that the harness let Claude infer backend capability from an offer table and - turn uncertainty into a false statement: "both backends are container-only". -- Once it made that false inference, it optimized back to the cheaper container-style service path. - -Fix made after this aborted run: - -- Endpoint prompt and `dstack-prototyping` now say not to infer "container-only" or "no reusable - state" from offers alone. -- Fleet state matters: `nodes: 0..N`, `idle_duration`, or an idle/running instance may keep capacity - warm for a later task or service. -- If backend reuse/SSH/cache behavior is uncertain, the agent must record uncertainty and choose an - experiment that can resolve it; it must not treat uncertainty as proof that a task would preserve - nothing. -- For create-recipe endpoints, "the model is small" or "the framework is common" is not enough to - skip a task/dev probe on an unverified fleet/backend/runtime path. - -## 2026-07-07 probe-quality rerun aborted after batch-style task probe - -Goal: rerun the same mixed RunPod/JarvisLabs scenario after the backend-capability prompt fix and -watch whether the agent chooses a reusable/inspectable placement and a meaningful probe. - -Project and setup: - -```text -project: endpoint-agent-reasoning -endpoint: qwen25-probe-quality-2152 -model: Qwen/Qwen2.5-0.5B-Instruct -fleet: probe-quality-mixed -``` - -Fleet-filtered offers again included cheaper RunPod A5000/L4 options and JarvisLabs L4: - -```text -runpod A5000 24GB $0.27/hr -runpod L4 24GB $0.39/hr -jarvislabs L4 24GB $0.44/hr -runpod RTX3090 $0.46/hr -``` - -What improved: - -- Claude did not default back to the cheapest RunPod service. -- Claude submitted a task first: `qwen25-probe-quality-2152-1`. -- The task landed on JarvisLabs `L4-1x` at `$0.44/hr`. -- The intended checks were deeper than `nvidia-smi`: vLLM/Torch/CUDA import, local vLLM server - start, `/health`, `/v1/models`, and `/v1/chat/completions`. - -What was still wrong: - -- The task was a batch script, not an interactive probe. The full investigation was encoded into - one shell command chain instead of starting a long-lived task and attaching/SSH-ing into it. -- The task had no instance cache mounts. Run JSON confirmed: - -```text -configuration.volumes=[] -job_spec.volumes=[] -runtime.volume_names=[] -``` - -- The agent still wrote `jarvislabs+runpod (container-style)` too loosely. It did not use that claim - to skip the probe this time, but backend capability still needs stronger evidence than names in an - offer table. - -Cleanup: - -```text -endpoint: qwen25-probe-quality-2152 stopped -task: qwen25-probe-quality-2152-1 terminated_by_user -backend: jarvislabs L4-1x, $0.44/hr -fleet: probe-quality-mixed deleted -``` - -Fix made after this run: - -- Endpoint prompt and `dstack-prototyping` now say that when SSH/attach is available, a task probe - should be long-lived (`sleep infinity` or equivalent) and inspected through attach/SSH; batch task - commands are only for unavailable attach/SSH or truly one-shot checks. -- Prompt/skill now require optional instance cache mounts for Hugging Face-style model caches when - useful, e.g. `/dstack-cache/huggingface` to `/root/.cache/huggingface` with `optional: true`, and - they require the agent to record why cache mounts were omitted. - -Checks after the fix: - -```text -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -``` - -Observed result: - -```text -31 passed -117 passed, 50 skipped -``` - -## 2026-07-07 probe-quality rerun aborted after cached batch task - -Goal: rerun after the long-lived-interactive-probe and cache-mount prompt update. - -Project and setup: - -```text -project: endpoint-agent-reasoning -endpoint: qwen25-probe-quality-2202 -model: Qwen/Qwen2.5-0.5B-Instruct -fleet: probe-quality-mixed -``` - -What improved: - -- The generated probe task included an optional Hugging Face instance cache mount: - -```yaml -volumes: - - instance_path: /dstack-cache/huggingface - path: /root/.cache/huggingface - optional: true -``` - -- The task again planned recipe-level checks beyond `nvidia-smi`: vLLM import, local vLLM server - start, health check, and local chat completion. - -What was still wrong: - -- The task was still a batch `commands` chain, not a long-lived task plus attach/SSH inspection. -- Claude again described `jarvislabs+runpod` as `container-style` based on fleet/offers instead of - grounded backend capability. -- It selected/submitted RunPod L4 instead of the slightly more expensive JarvisLabs path, so the - backend-placement behavior regressed from the previous run. -- The run was stopped immediately because it no longer tested the intended interactive probe path. - -Cleanup: - -```text -endpoint: qwen25-probe-quality-2202 stopped -task: qwen25-probe-quality-2202-1 terminated_by_user -backend: runpod NVIDIA L4 -fleet: probe-quality-mixed deleted -``` - -Conclusion: - -Prompt/skill prose is no longer enough for this part of the harness. The next fix should be -structural: pass explicit backend/fleet capability context from the server into the workspace and -consider a pre-submit guard or required pre-submit artifact when the prompt expects an interactive -probe. More wording alone is unlikely to make Claude reliably choose attach/SSH over a batch task. - -## 2026-07-07 existing functionality e2e - -Goal: before moving to the next endpoint-plan item, re-check the already implemented surfaces -without spending GPU time. - -Project state: - -```text -project: endpoint-e2e -fleet: endpoint-e2e-gpu active, nodes 0..1, no live instances -unfinished endpoints: none -latest visible run row: qwen25-broad-fleet-1, terminated/stopped_by_user -``` - -Read-only checks: - -- `dstack endpoint --project endpoint-e2e -a -n 10` showed stopped endpoint history and no - unfinished endpoint. -- `dstack endpoint get qwen25-broad-fleet --json` returned `stopped`, no linked run/url/error. -- `dstack endpoint logs qwen25-broad-fleet --since 24h` showed endpoint progress lines only, not - backing service logs. -- `dstack endpoint logs qwen25-log-start-smoke --since 24h` showed the server startup line plus - Claude's first progress line. -- top-level `dstack logs qwen25-broad-fleet` and `dstack stop qwen25-broad-fleet -y` looked for a - run and returned `Run qwen25-broad-fleet not found`, so endpoint/run command separation holds. -- `dstack endpoint stop qwen25-broad-fleet -y` returned `Endpoint qwen25-broad-fleet is already - stopped`. - -Preset checks: - -- compact `dstack endpoint preset list` grouped recipes by model and showed GPU only. -- verbose `dstack endpoint preset list -v` showed full scheduling resources. -- `dstack endpoint preset get Qwen/Qwen2.5-0.5B-Instruct --json` returned both recipes with - `validations`. -- `dstack endpoint preset get ...` without `--json` correctly refused with `Use --json to output - the endpoint preset.` - -Apply previews, all answered `n`: - -- `qwen25-broad-reuse-preview.dstack.yml` selected preset model - `Qwen/Qwen2.5-0.5B-Instruct`, recipe `4a73893b`, and showed 10 matching RunPod offers. -- `qwen25-broad-fleet-harness.dstack.yml` showed the create/agent path and confirmation without - submitting. -- `no-fleet-create-preview.dstack.yml` against project `endpoint-agent-choice` showed `The project - has no fleets. Create one before submitting an endpoint.` -- old `qwen25-05b-create.dstack.yml` correctly showed `No fleets match the endpoint configuration` - because it still references deleted fleet `endpoint-e2e-runpod`. - -Issue found and fixed: - -- `dstack endpoint preset list -v` worked, but `dstack endpoint preset -v list` printed compact - output even though the help shape allowed `-v` before the action. Root cause: the `list` - subparser's default `verbose=False` overwrote the parent parser's `verbose=True`. Fixed by making - the child default suppressed; both forms now print verbose output. - -Checks run: - -```text -uv run pytest src/tests/_internal/cli/commands/test_endpoint.py -q -uv run pytest src/tests/_internal/cli/commands/test_endpoint.py src/tests/_internal/cli/utils/test_preset.py src/tests/_internal/cli/utils/test_endpoint.py src/tests/_internal/cli/services/configurators/test_endpoint.py -q -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py::TestFindMatchingPreset src/tests/_internal/server/services/endpoints/test_claude_agent.py::TestClaudeAgentService::test_agent_cli_guard_explains_singular_run_config_fields -q -uv run ruff check src/dstack/_internal/cli/commands/endpoint.py src/tests/_internal/cli/commands/test_endpoint.py -``` - -Result: existing no-spend endpoint functionality looks coherent after the CLI verbose fix. The main -remaining plan work is not basic CLI behavior; it is still the agent/harness durability and richer -real-run scenarios. - -## 2026-07-07 main loop validation - -Goal: stop polishing peripheral CLI behavior and validate the central loop again: - -```text -endpoint config -> Claude deploys and verifies service -> endpoint running -> preset saved -> reuse preview selects the learned recipe -``` - -Config was kept outside the repo: - -```text -~/dstack-endpoints-demo/e2e-2026-07-07/qwen25-mainloop-1642.dstack.yml -``` - -Endpoint: - -```yaml -type: endpoint -name: qwen25-mainloop-1642 -model: Qwen/Qwen2.5-0.5B-Instruct -preset_policy: create -fleets: - - endpoint-e2e-gpu -spot_policy: auto -max_price: 0.5 -``` - -What happened: - -- Claude inspected the allowed fleet and current offers, then submitted - `qwen25-mainloop-1642-1` with `uv pip install vllm`. -- That first service landed on RunPod RTX 2000 Ada at `$0.24/hr`, reached `running`, then failed at - CUDA initialization. vLLM `0.24.0` pulled a CUDA 13 torch build, while the host driver exposed - CUDA 12.8 (`found version 12080`). -- Claude diagnosed the failure from service logs, recorded it in endpoint progress, and submitted - `qwen25-mainloop-1642-2` using - `vllm/vllm-openai:v0.24.0-cu129-ubuntu2404`. -- The second service reached `running`; `/v1/models` and a real `/v1/chat/completions` request both - returned HTTP 200 with model `Qwen/Qwen2.5-0.5B-Instruct`. -- The server consumed `final_report.json`, linked endpoint `service_run_id`, set the endpoint to - `running`, saved the preset, and `dstack endpoint stop` terminated the backing run. - -Important harness observations: - -- This was a useful real failure: unpinned package installs are risky on container backends because - host driver/CUDA compatibility is not visible in offers. -- Endpoint progress was substantially better than earlier traces: it showed the failed hypothesis, - the driver mismatch, the second attempt, and the final verification. -- The official pinned vLLM image avoided the CUDA mismatch, but it also made the provisioning/startup - phase slower. The agent handled this with bounded polls, though it could still write a progress - line during long image pulls. -- The final service YAML was vendor-aware (`gpu: nvidia:16GB..24GB:1`) and honored the endpoint - fleet/price/spot constraints. - -Preset reuse issue found and fixed: - -- Before the fix, a no-spend `preset_policy: reuse` preview for the same model selected older - recipe `4a73893b`, not the freshly verified recipe `01141556`. -- Root cause: `save_preset` appended new recipes after older recipes, while v1 planning intentionally - uses the first provisionable recipe. -- Fix: saving an incoming learned recipe now moves that recipe to the front of the model preset. - The planner remains simple: first stored recipe with offers wins. -- After re-saving the successful recipe through the preset service, the same no-spend reuse preview - selected recipe `01141556` and showed matching RunPod offers without invoking Claude. - -Checks run: - -```text -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py -q -``` - -Result: the main loop works for this RunPod/Qwen/vLLM case, including recovery from one real runtime -failure and immediate no-agent preset reuse at the apply-plan level. Next main-path work should focus -on learned recipe quality and more representative scenarios, not more CLI surface polish. - -Paid reuse follow-up: - -- Submitted `qwen25-reuse-after-mainloop` with `preset_policy: reuse`. -- Apply selected recipe `01141556`, the freshly verified CUDA-12 vLLM image recipe. -- The service run landed on RunPod CA-MTL-1, NVIDIA RTX A5000 24GB, `$0.27/hr`. -- Provisioning looked stuck for several minutes from dstack alone: endpoint/run stayed - `provisioning` and normal workload logs were initially empty. -- RunPod native REST/GraphQL showed useful intermediate state: the pod was rented, desired status was - `RUNNING`, runtime/ports existed, image was `vllm/vllm-openai:v0.24.0-cu129-ubuntu2404`, and the - machine was RTX A5000. The native API did not expose image-pull percentage through the fields used - here. The HAPI logs URL returned 401/403 with the configured backend API key, so it is not a stable - backend-diagnostic path for dstack. -- The endpoint then reached `running`, service logs showed vLLM startup and repeated HTTP 200 model - requests, and `dstack endpoint stop` terminated the backing run cleanly. - -Diagnostic lesson: when dstack is stuck at `provisioning` with no workload logs, official native -backend APIs are worth checking for pod/runtime/container state. Do not depend on web-app session -tokens or Clerk endpoints; use the backend's configured API credential and print only sanitized -operational fields. - -## 2026-07-07 7B main-loop validation - -Goal: validate the central learning loop on a less trivial public model: - -```text -endpoint config -> Claude chooses hardware and deploys -> service fails for real reasons -> -Claude adjusts -> service answers model API -> endpoint running -> preset saved -> reuse preview -selects the preset without Claude -``` - -Config was kept outside the repo: - -```text -~/dstack-endpoints-demo/e2e-2026-07-07/qwen25-7b-mainloop-1722.dstack.yml -``` - -Endpoint: - -```yaml -type: endpoint -name: qwen25-7b-mainloop-1722 -model: Qwen/Qwen2.5-7B-Instruct -preset_policy: create -fleets: - - endpoint-e2e-gpu -spot_policy: auto -max_price: 0.5 -``` - -What happened: - -- Claude chose a NVIDIA 24GB..48GB envelope for the 7B model, avoiding the earlier too-loose - `gpu: 1` pattern. -- Run `qwen25-7b-mainloop-1722-1` failed before capacity with `failed_to_start_due_to_no_capacity`. -- Run `qwen25-7b-mainloop-1722-2` landed on RunPod A40 and then failed because FlashInfer tried - runtime JIT through `/usr/local/cuda/bin/nvcc`, which was not present. -- Run `qwen25-7b-mainloop-1722-3` switched to `vllm/vllm-openai:v0.24.0`, landed on RunPod - CA-MTL-1 NVIDIA RTX A5000 at `$0.27/hr`, and served the model successfully. -- Final verification used the model endpoint, not service status alone: `/v1/models` returned - `Qwen/Qwen2.5-7B-Instruct` with `max_model_len: 32768`, and `/v1/chat/completions` returned - HTTP 200 with content `dstack verification OK`. -- The endpoint moved to `running`, linked `qwen25-7b-mainloop-1722-3`, saved a project-scoped - preset, and was stopped afterward to cap spend. The backing run is terminated. - -Saved preset: - -```text -path: ~/.dstack/server/projects/endpoint-e2e/presets/qwen-qwen2-5-7b-instruct.dstack.yml -recipe: cb373545 -scheduling cpu=2.. mem=8GB.. disk=100GB.. gpu=nvidia:24GB..48GB:1 -tested: cpu=9 mem=50GB disk=100GB gpu=A5000:24GB:1 -image: vllm/vllm-openai:v0.24.0 -``` - -No-spend reuse preview: - -```text -config: qwen25-7b-reuse-preview.dstack.yml -policy: reuse -selected: Qwen/Qwen2.5-7B-Instruct recipe cb373545 -offers: 12 matching offers under endpoint-e2e-gpu / max_price $0.5 -submitted: no -``` - -Important observations: - -- The agent made a reasonable hardware decision for a 7B model and preserved the broad scheduling - envelope separately from exact A5000 validation hardware. -- The loop recovered from both no-capacity and a concrete runtime/JIT failure without server-side - special cases. -- The first full service attempt still installed/used a runtime path that led to avoidable JIT - friction. For vLLM OpenAI services, the prompt/skill should bias toward pinned official vLLM - serving images unless there is a specific reason to prototype package installation. -- The endpoint worker correctly waited for the agent's functional verification report before - marking the endpoint running. -- Reuse planning is proven for this learned 7B preset; paid reuse deployment was not submitted in - this step. - -## 2026-07-07 Qwen3.6-27B main-loop validation - -Goal: push the learning loop beyond small Qwen models and verify that the agent can use current -recipe sources, choose a defensible 80GB single-GPU shape, wait through a long model startup, perform -real model API verification, and save a reusable recipe. - -Config was kept outside the repo: - -```text -~/dstack-endpoints-demo/qwen27-2026-07-07/qwen36-27b-create.dstack.yml -``` - -Endpoint: - -```yaml -type: endpoint -name: qwen36-27b-mainloop -model: Qwen/Qwen3.6-27B - -preset_policy: create -fleets: - - qwen27-runpod-a100 -spot_policy: auto -max_price: 1.6 -``` - -Result: - -- Endpoint reached `running`. -- Final service run: `qwen36-27b-mainloop-1`. -- Run id: `34ed301f-f1f3-4f8d-bebc-5ecae99568b6`. -- Backend/hardware: RunPod `CA-MTL-3`, NVIDIA A100 80GB PCIe, 31 CPU, 117GB RAM, 200GB disk. -- Price: `$1.39/hr`, within endpoint `max_price: 1.6`. -- Image: `vllm/vllm-openai:v0.24.0`. -- Service resource envelope saved in the recipe: `gpu: nvidia:80GB:1`, `disk: 200GB`. -- Preset list now shows model `Qwen/Qwen3.6-27B` with GPU `nvidia:80GB:1`. -- The endpoint was stopped after verification to cap spend; the backing run terminated cleanly with - final observed cost about `$0.3233`. - -Agent evidence: - -- Read Hugging Face config and vLLM recipe sources for the exact model. -- Identified `Qwen3_5ForConditionalGeneration`, BF16, hybrid GDN linear-attention + full-attention, - multimodal vision encoder, and the official vLLM recipe floor. -- Chose BF16 TP1 on 80GB instead of FP8/NVFP4 because the allowed fleet was A100 and the recipe's - FP8/NVFP4 paths are not the right Ampere assumption. -- Submitted the service directly and recorded why: RunPod is container-style, a task probe would not - reliably preserve the 50GB+ model download for the final service, and the main remaining unknowns - were visible in service startup logs. -- Waited through weight download, model loading, torch compile, warmup, CUDA graph capture, API server - startup, and dstack service probe success. -- Verified the model through the OpenAI-compatible chat API. The successful request returned HTTP 200, - response model `Qwen/Qwen3.6-27B`, `finish_reason=stop`, and content - `The capital of France is Paris.` - -Runtime evidence from logs and verification artifacts: - -- Weight download took about 129 seconds. -- Model loading took 51.1 GiB GPU memory. -- Available KV cache was 18.56 GiB, with 281,804 KV-cache tokens and reported maximum concurrency - 8.60x at 32,768 tokens. -- vLLM used Triton/FLA GDN linear-attention, FlashAttention v2, and FlashInfer sampling without CUDA - or OOM errors. - -Harness observations: - -- The `progress` helper materially improved endpoint logs. The user can now see recipe/hardware - reasoning, why service-first was chosen, when the run landed capacity, when model loading passed, - when API verification started, and when the final report was written. -- The prompt is still long, but the agent followed the core constraints: existing fleet only, - endpoint run names as `-`, final functional verification, and - project-scoped preset save. -- One artifact-writing bug appeared: the final report text had `.39/hr` instead of `$1.39/hr` because - the agent wrote JSON via an unquoted shell heredoc, allowing shell expansion of `$1`. The prompt and - `dstack-prototyping` skill now require Python serializers or quoted heredocs such as `<<'EOF'` for - artifacts. -- The agent still used `head -c` once during source inspection despite the prompt. It did not break - the run, but the harness should continue pushing source inspection toward parsed/bounded fields. -- This run does not prove the task/dev-environment-first path because the allowed fleet was - container-style RunPod. We still need a VM/SSH-fleet scenario where a task or interactive experiment - should clearly be cheaper and more informative than repeated services. - -No-spend reuse preview after stopping: - -```text -config: qwen36-27b-reuse-preview.dstack.yml -policy: reuse -selected: Qwen/Qwen3.6-27B recipe 4a0dd437 -offers: 14 matching RunPod A100 80GB offers under qwen27-runpod-a100 / max_price $1.6 -submitted: no -``` - -Checks run after the prompt/skill updates: - -```text -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -``` - -Result: - -```text -31 passed -``` - -## Probe Task Shape Guard - -The `qwen25-probe-quality-*` live tests showed that prompt wording alone did not reliably produce -the intended interactive task path. One run skipped the task and optimized back to RunPod service -first; the next selected a task and JarvisLabs, but encoded the entire serving investigation as one -batch command. That proved the useful next fix was not more prose, but a structural guard. - -Implemented guard: - -- Endpoint agent task probes must be long-lived attach/SSH targets, normally `commands: - [sleep infinity]`. -- The wrapper rejects batch probe commands that contain the serving investigation inside task YAML: - host checks, Python/framework imports, vLLM/SGLang startup, or curl/API probes. -- The agent should run those checks after attaching/SSHing into the live task. - -Checks after the guard: - -```text -uv run pytest src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -33 passed - -uv run pytest src/tests/_internal/server/services/test_endpoint_presets.py src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -119 passed, 50 skipped - -uv run ruff check src/dstack/_internal/server/services/endpoints/agent/claude.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -All checks passed -``` - -Next live e2e should specifically check whether Claude reacts to the wrapper correction by creating -a long-lived task, attaching/SSHing into it, running real serving checks inside, and only then -promoting a clean service. - -## Backend Docs Gate And Endpoint Log Watch - -Config: - -```text -project: endpoint-agent-reasoning -config: qwen25-docs-gate-e2e.dstack.yml -model: Qwen/Qwen2.5-0.5B-Instruct -fleet: probe-quality-guard -policy: create -``` - -Result: - -- Claude classified backends against `https://dstack.ai/docs/concepts/backends.md`, chose JarvisLabs - as VM-based, and rejected RunPod as container-based/no instance volumes. -- The task config used `commands: [sleep infinity]`, `fleets: [probe-quality-guard]`, - `backends: [jarvislabs]`, `gpu: nvidia:16GB..`, and optional Hugging Face instance cache. -- Claude attached/SSHed into the task, started vLLM inside it, and verified a local - `/v1/chat/completions` request. -- Claude stopped the task, waited for the instance to become idle, submitted the same recipe as a - service on JarvisLabs, and verified `/v1/chat/completions` through the dstack service URL. -- The endpoint reached `running`, linked `qwen25-docs-gate-e2e-2`, wrote a schema-clean - `final_report.json`, and was then stopped cleanly. - -CLI/log UX change validated by this run: - -- `dstack endpoint logs -w` was added. -- Foreground endpoint apply now polls endpoint status and streams the same endpoint progress log - stream without using `attach`. -- Endpoint logs print local timestamps because they are progress events, unlike raw streamed run logs. -- Endpoint log following uses an overlap plus duplicate counts, so records sharing the same - timestamp are not swallowed at poll boundaries. - -Checks: - -```text -uv run pytest src/tests/_internal/cli/commands/test_endpoint.py src/tests/_internal/cli/services/configurators/test_endpoint.py src/tests/_internal/server/services/endpoints/test_claude_agent.py -q -42 passed - -uv run ruff check src/dstack/_internal/cli/commands/endpoint.py src/dstack/_internal/cli/services/configurators/endpoint.py src/dstack/_internal/cli/services/endpoint_logs.py src/tests/_internal/cli/commands/test_endpoint.py -All checks passed -``` - -Remaining issue: - -- Endpoint logs are now much more useful, but the first `dstack apply` in this run was started - before the foreground-log-streaming CLI change, so the new attached-apply UX still needs a direct - fresh smoke after this patch. diff --git a/endpoint-implementation-plan.md b/endpoint-implementation-plan.md deleted file mode 100644 index a835dbddad..0000000000 --- a/endpoint-implementation-plan.md +++ /dev/null @@ -1,921 +0,0 @@ -# Implementation plan: `endpoint` configuration type - -Status: **implementation in progress**. This document started as a research-backed implementation plan verified against `master` @ `28ea5f86f` (2026-07-03); it is now also used as the implementation tracker. Line numbers drift; symbol names are the anchor. - -## Project thesis - -The endpoint feature is not primarily a new CRUD resource. It is a **learning deployment loop**: - -1. A user asks dstack to serve a model. -2. If dstack already has a tested preset for that model, it should submit a normal dstack service from that preset and mark the endpoint running when the service is ready. -3. If dstack does not know a working deployment yet, a real agent investigates, uses dstack like a power user, deploys and verifies a final service, and saves what worked as a preset. -4. The next compatible endpoint for that model should be able to use the preset path without paying the agent again. - -The preset path and the agent path must converge. If the agent can make a model work once but saves a preset that is too narrow, unreproducible, missing evidence, or unusable without the agent, the feature has not solved the real problem. - -Non-negotiable project invariants: - -- A preset is a verified deployment artifact, not just a service YAML. -- Each preset is keyed by model and contains one or more deployment recipes. A recipe's `service` contains the scheduling resources used for reuse/offer matching; `validations` store exact verified hardware evidence grouped in the same order as the service replica groups. -- The server owns endpoint state, locking, run linking, stopping/cleanup, and preset persistence. The agent owns deployment investigation and final functional verification. -- The server must not mark an agent-created endpoint running based only on its own service-readiness bookkeeping; the agent must report that the requested model was actually served and answered a real model request. -- Work should be driven by real deployment failures. Avoid adding abstractions before live runs show they are needed. - -## 0. Implementation checklist and plan deltas - -Keep this section current while implementing. It is the short operational view of the longer plan below: what is already in the working tree, what is still missing for v1, and where the implementation intentionally diverged from the original plan after code/UX review. - -### Done - -- [x] Research artifacts copied into the repo under `endpoint-implementation-research/`. -- [x] Endpoint core/API/CLI skeleton: `type: endpoint`, `EndpointConfiguration`, endpoint REST schemas/router/client group, `dstack endpoint list|get|logs|stop|preset`, and `dstack apply` support. -- [x] Endpoint DB/pipeline skeleton: `EndpointModel`, migration, pipeline registration, durable locking, fetch/worker flow, events, and status transitions. -- [x] Apply-plan UX aligned with run apply: one stable property table, preset policy, selected preset/offers when present, no `Model`/`Action`/`Service`/`Agent`/custom provisioning section. -- [x] Endpoint plan displays the configured/default `preset_policy` (`reuse-or-create` by default), not the effective fallback path. -- [x] Preset-backed provisioning path: local presets are matched through normal run planning, service runs are submitted through the existing run apply path, and `creation_policy` is not forced to `reuse`. -- [x] Preset planning distinguishes a model-matched preset from a provisionable preset. Preset submission requires available offers for every planned job; `reuse-or-create` can fall through to the agent path when only non-provisionable presets match. -- [x] Endpoint lifecycle safety: conflict-checked submission, server-side stop that stops the linked backing run, and failed-endpoint teardown of the linked non-terminal run. -- [x] Endpoint apply treats any terminal same-name endpoint as finished: no stop prompt; create resets the same endpoint row back to `submitted`, and only non-terminal same-name endpoints require stop/override. -- [x] Preset-backed endpoint submission handles its deterministic service-run-name conflicts like run apply: non-terminal conflicting runs fail before submission, while terminal conflicting runs are left to the existing run submission path to recycle. -- [x] Endpoint readiness uses backing service readiness only: run is `RUNNING`, has a registered running job, and exposes `ServiceSpec.model.base_url`; no extra endpoint probe in v1. -- [x] Preset storage/parsing: `EndpointPresetService`, project-local implementation under `/projects//presets`, `endpoint-preset` YAML wrapper, model-keyed `recipes`, ordered `validations`, legacy `replica_spec_groups` conversion, validation, merge-by-model list/get/save/delete, env value redaction, and literal non-secret env preservation. -- [x] Learned preset plumbing: build or merge a model-level preset from a ready service run, preserve the service recipe/resources as the scheduling contract, record exact running replica resources in `validations`, and save it when an agent-provisioned endpoint becomes running. -- [x] Endpoint preset CLI: `dstack endpoint preset list|get --json|delete`. The compact list groups by model and shows only the scheduling GPU; `-v` shows full service scheduling resources. Child `recipe=N` rows appear only when a model has multiple recipes; internal recipe ids and exact verified hardware stay in `get --json`. -- [x] Endpoint list UX now follows the `ps` convention more closely: default output shows unfinished endpoints, or the latest finished endpoint only when nothing is active; `POLICY` is shown by default and the backing service run is only shown in verbose/detail/JSON output. -- [x] Endpoint status names adjusted: `prototyping` while the server agent investigates/deploys, `running` for a ready endpoint, `stopping`/`stopped` for endpoint stop. The runtime `active` alias was removed; local prototype DB rows should be cleaned directly. -- [x] Endpoint UX split from run UX: `dstack endpoint logs` reads endpoint progress logs, `dstack endpoint stop` stops endpoints, top-level `dstack logs`/`dstack stop` remain run-only, and top-level `dstack preset` was removed. -- [x] Running endpoint apply output renders known relative proxy URLs as absolute URLs using the configured API server URL. -- [x] Endpoint migration is safe for Postgres partial indexes and boolean server defaults. -- [x] Agent lifecycle skeleton: `AgentService` abstraction, disabled default, agent settings, `EndpointPlan.provisioning_plan=type:"agent"` when an enabled agent is injected, fake-agent pipeline integration, and a v0 Claude Code subprocess runtime. -- [x] Agent unavailable UX distinguishes a genuinely missing `DSTACK_AGENT_ANTHROPIC_API_KEY` from a configured key with no real agent implementation registered yet. -- [x] Endpoint run identity uses `EndpointModel.service_run_id` for the current/latest service run and `EndpointRunSubmissionModel` for ordered endpoint-submitted run history. -- [x] Structured final-report validation for the endpoint pipeline contract. -- [x] Endpoint-scoped workspace/process handling added as part of the v0 Claude Code runtime, not as standalone scaffolding. -- [x] Raw Anthropic Messages-loop prototype was removed. It was the wrong abstraction; the endpoint agent must use a real agent runtime, not a hand-rolled tool loop. -- [x] Packaged endpoint-agent prompt/context resources: `resources/system_prompt.md` plus repo-root `skills/dstack` and `skills/dstack-prototyping`, force-included into wheels/sdists and copied into each Claude workspace. -- [x] First full learning-loop proof: Claude-created Qwen preset was reused later by `preset_policy: reuse` without invoking Claude; the endpoint reached `running`, answered a model request, and stopped cleanly. -- [x] Fresh create-policy Claude run: `Qwen/Qwen2.5-0.5B-Instruct` on RunPod reached `running`, verified a real OpenAI-compatible chat request, saved preset `qwen-qwen2-5-0-5b-instruct-e30ddb94`, and stopped the backing run cleanly. -- [x] No-cost reuse preview for the fresh Qwen2.5 preset selected the learned preset and found 5 matching RunPod offers under the endpoint constraints. -- [x] Paid reuse deployment for the fresh Qwen2.5 preset reached `running` without Claude, answered a model request on RunPod A5000, and stopped cleanly. -- [x] Fresh create-policy rerun after prompt/env fixes exposed a real harness weakness: the agent stopped using `~/.dstack/config.yml` and append-only submission records worked, but it repeatedly submitted full services on a container backend to learn runtime facts that should be prototyped with a dev environment or small task when an allowed reusable VM/SSH fleet exists. -- [x] Stop-time agent cancellation implemented after the live leak: `dstack endpoint stop` now asks the agent runtime to abort, terminates the stored Claude process group, keeps the endpoint `stopping` if the agent process is on another host, and still stops linked/submitted runs through `EndpointRunSubmissionModel`. -- [x] Stop-time cancellation live validation: stopping before Claude starts and stopping after Claude starts both reached `stopped` with no endpoint-submitted runs left. A zombie-only process-group issue was found and fixed during validation. -- [x] Historical create-policy run in an isolated project with no fleets proved the real agent loop could deploy and verify a model, but also showed an unwanted contract: Claude created an endpoint-scoped fleet. That behavior is now rejected; fleet creation is a user/admin step before endpoint submission. -- [x] No-fleet contract validation: with no active fleet, endpoint apply now shows "The project has no fleets. Create one before submitting an endpoint." for any `preset_policy`; forced detached apply exits `1` and does not resubmit a terminal endpoint. The pipeline also fails already-submitted endpoints with the same no-fleets status before preset or Claude processing. -- [x] Existing-fleet contract validation: after creating zero-target fleet `endpoint-agent-reasoning-fleet`, the endpoint plan returned the agent path. Claude started, inspected the existing fleet, correctly ignored prior stopped run `qwen25-7b-vm-reasoning-1` as old/different-fleet work, and used `dstack offer --fleet endpoint-agent-reasoning-fleet ...`. The run was stopped before any new service submission or GPU spend. -- [x] Endpoint progress stream no longer prescribes labels/categories/templates: the server accepts plain text progress lines or JSON objects with a `message` string and displays only the message text. -- [x] Harder existing-fleet e2e exposed prompt-only enforcement failure: Claude correctly used `dstack offer --fleet endpoint-agent-reasoning-fleet`, then wrote a probe task without `fleets` and submitted it. The task failed with `no offers` before provisioning (`cost: 0.0`), and the endpoint/agent were stopped. A workspace-local `dstack` CLI guard now blocks fleet apply and blocks agent-submitted run configs that omit or violate explicit endpoint `fleets`. -- [x] Harder existing-fleet e2e after the CLI guard succeeded end-to-end: `Qwen/Qwen2.5-7B-Instruct` deployed on existing fleet `endpoint-agent-reasoning-fleet`, selected Jarvis L4:24GB at `$0.44/hr`, served through `vllm/vllm-openai:v0.24.0`, answered a real `/v1/chat/completions` request, saved project preset `qwen-qwen2-5-7b-instruct-d79aa1d8`, and stopped cleanly. The test fleet was deleted afterward to stop idle spend. -- [x] Guard behavior validated in the live run: Claude first wrote a service config without `fleets`, the workspace-local `dstack` shim rejected it, Claude corrected the YAML to include `fleets: [endpoint-agent-reasoning-fleet]`, previewed/applied it, and the final saved preset remained usable without Claude. -- [x] Learned-preset reuse preview validated for the 7B run: while the original service occupied the only `nodes: 0..1` fleet slot, reuse correctly matched the preset but showed no offers; after stopping the endpoint, the same preview selected the preset and showed the idle Jarvis L4 offer without invoking Claude. -- [x] Agent CLI guard now enforces the default "all existing project/imported fleets" path by passing discovered active usable fleet names into the workspace shim, and rejects `dstack offer` / `dstack apply` calls that omit allowed fleets or violate accepted endpoint profile constraints such as backend, region, instance type, spot policy, max price, durations, instances, tags, and backend options. -- [x] Fresh RunPod existing-fleet e2e after the guard: `Qwen/Qwen2.5-1.5B-Instruct` with `preset_policy: create`, `backends: [runpod]`, `fleets: [endpoint-e2e-runpod]`, `spot_policy: on-demand`, and `max_price: 0.5` reached `running`. Claude first omitted `fleets` in the service YAML, the workspace shim rejected the preview, Claude corrected the config, the first service submission failed with transient RunPod `failed_to_start_due_to_no_capacity`, the second submission added retry, landed RTX 2000 Ada at `$0.24/hr`, answered `/v1/chat/completions`, saved preset `qwen-qwen2-5-1-5b-instruct-4d68b0a3`, and `dstack endpoint stop` terminated the backing run cleanly. -- [x] Immediate reuse preview for that saved Qwen2.5-1.5B preset validated both capacity states: while the only allowed RunPod fleet slot was occupied, the plan selected the preset but showed no offers; after stopping the endpoint, the same reuse preview selected the preset and showed 7 matching offers without invoking Claude. -- [x] Prompt/skill tightened from the fresh trace: final service YAML must carry applicable endpoint constraints, progress must report run-capacity/running/final-verification milestones, polling/probe commands must stay bounded to avoid Claude Code Bash timeouts, and `final_report.json` must stay limited to the JSON-schema fields while hardware/driver/reasoning evidence lives in workspace artifacts. -- [x] Fresh validation run after prompt/skill tightening: `Qwen/Qwen2-0.5B-Instruct` with `preset_policy: create`, `backends: [runpod]`, `fleets: [endpoint-e2e-runpod]`, `spot_policy: on-demand`, and `max_price: 0.5` reached `running`, answered `/v1/chat/completions`, saved preset `qwen-qwen2-0-5b-instruct-532ddf05`, and stopped cleanly. Improvements were real: the first service YAML included `fleets`, endpoint logs included running and verification milestones, polling was bounded instead of concurrent watchers, and `final_report.json` was schema-clean. Remaining harness/preset quality issues from the same trace: the service requested `gpu: 1` without a GPU-memory floor, wrote a transient `"run_id":"pending"` submission record, waited on a Uvicorn log marker after an early proxy probe, and still did not capture a direct `nvidia-smi` driver string. -- [x] Immediate reuse preview for the saved Qwen2-0.5B preset validated both capacity states: while the only allowed RunPod fleet slot was occupied, the plan selected the preset but showed no offers; after stopping the endpoint, the same reuse preview selected the preset and showed 7 matching offers without invoking Claude. -- [x] Removed the premature agent-budget surface from v1: no endpoint `max_agent_budget`, no `DSTACK_AGENT_ANTHROPIC_MAX_BUDGET_USD`, no DB/session spend fields, no CLI plan row, no API plan `max_budget`. Cost/budget governance is Later and must be added only with durable per-session accounting. -- [x] Tightened the endpoint system prompt and `dstack-prototyping` skill after the `gpu: 1` trace: final LLM services should normally request a defensible GPU memory/count envelope, and count-only GPU scheduling must be intentional and recorded in hardware reasoning. -- [x] Agent session restart semantics tightened: a server restart or dead same-host Claude process resumes the same endpoint agent session/workspace instead of creating a new session; a new endpoint lifecycle from re-apply creates the next session and writes `previous_sessions.md` so the agent can use older session evidence read-only. -- [x] Same-host restart and minimum-resource guidance validated in a fresh paid RunPod run: `Qwen/Qwen2-0.5B-Instruct` stayed in the same agent session across a local server restart, did not spawn a duplicate Claude process, deployed `qwen2-05b-restart-1110-1`, verified `/v1/chat/completions`, saved a model-level preset recipe, and stopped cleanly. The final service YAML used `gpu: 16GB..24GB:1`; exact RTX 3090 placement belongs under recipe `validations`. -- [x] Endpoint agent stdout is no longer copied to endpoint logs. Endpoint logs are populated from `progress.jsonl` and explicit server lifecycle messages; Claude assistant chatter and final markdown summaries remain in trace/artifacts. -- [x] Endpoint agent logs now start promptly: the server writes an immediate progress line when a detached Claude session starts, stores it in `progress.jsonl`, advances the progress offset to avoid replay, and the prompt requires Claude's first workspace action to be a short progress message before inspecting state/offers/docs. Live smoke `qwen25-log-start-smoke` showed the startup line and Claude's first progress line, then was stopped before any service submission. -- [x] Follow-up paid Qwen2.5 preset-reuse deployment validated the no-Claude path after the model-level recipe format: `Qwen/Qwen2.5-0.5B-Instruct` with `preset_policy: reuse` selected recipe `4a73893b`, deployed on RunPod A5000, reached `running`, answered `/v1/chat/completions`, and stopped cleanly. -- [x] Local duplicate preset files are no longer a planning/list/get inconsistency: the local service merges files by model in memory, `get --json` returns all recipes for the model, `save` collapses duplicates into one canonical file, and `delete` removes all files for that model. -- [x] Broad-fleet vendor-aware harness check: endpoint config used neutral fleet `endpoint-e2e-gpu` with Lambda/Verda/JarvisLabs/RunPod and no endpoint `backends`; current matching offers under `$0.5/hr` were still all RunPod, Claude selected RunPod based on offers, wrote final service resources as `gpu: nvidia:16GB..24GB:1`, verified `/v1/chat/completions`, saved a vendor-aware recipe, and stopped cleanly. Reuse preview after freeing the `nodes: 0..1` fleet slot showed offers without invoking Claude. -- [x] Prompt/guard hardening after repeated YAML mistakes: endpoint prompt and `dstack-prototyping` now distinguish plural service YAML fields (`fleets`, `backends`, `regions`, `instance_types`) from singular CLI flags (`--fleet`, `--backend`, `--region`, `--instance-type`), and the workspace CLI guard returns actionable errors for singular run-config keys before a paid preview/apply can proceed. -- [x] Existing no-spend endpoint functionality e2e: endpoint list/get/logs/stop, endpoint-vs-run command separation, preset list/get, reuse/create/no-fleet apply previews, and preset verbose output. Fixed `dstack endpoint preset -v list` so parent `-v` is not overwritten by the child parser. -- [x] Main learning-loop validation after prompt/guard fixes: `Qwen/Qwen2.5-0.5B-Instruct` with `preset_policy: create` recovered from a real RunPod CUDA driver mismatch, submitted a pinned CUDA-12 vLLM image, verified `/v1/chat/completions`, linked the endpoint to the final run, saved a preset, stopped cleanly, and a subsequent `preset_policy: reuse` endpoint selected the freshly verified recipe, reached `running` without Claude, then stopped cleanly. -- [x] Learned recipe priority fixed: `save_preset` now writes incoming learned recipes before older recipes for the same model, preserving the simple "first recipe with offers wins" planner while ensuring the recipe just proven by the agent is what immediate reuse sees. -- [x] Harder 7B learning-loop validation: `Qwen/Qwen2.5-7B-Instruct` with `preset_policy: create` selected a NVIDIA 24GB..48GB envelope, recovered from no-capacity and a FlashInfer/nvcc runtime failure, deployed `vllm/vllm-openai:v0.24.0` on RunPod RTX A5000, verified `/v1/models` and `/v1/chat/completions`, saved a project-scoped recipe with exact A5000 validation evidence, stopped cleanly, and a no-spend `preset_policy: reuse` preview selected the learned recipe with matching offers without invoking Claude. -- [x] Larger-model learning-loop validation: `Qwen/Qwen3.6-27B` with `preset_policy: create` used current HF/vLLM recipe evidence, chose BF16 TP1 on `gpu: nvidia:80GB:1`, deployed `vllm/vllm-openai:v0.24.0` on RunPod A100 80GB at `$1.39/hr`, waited through 51.1 GiB model loading and CUDA graph capture, verified `/v1/chat/completions`, linked `qwen36-27b-mainloop-1`, and saved a project-scoped preset. The endpoint was stopped cleanly after verification, and a no-spend `preset_policy: reuse` preview selected the saved recipe with 14 matching offers without invoking Claude. This proved the real agent loop can handle a non-trivial current model, not just small smoke models. -- [x] Qwen3.6-27B trace hardened artifact-writing guidance: the agent verified the service correctly, but one final-report string was corrupted from `$1.39/hr` to `.39/hr` by an unquoted shell heredoc. The endpoint system prompt and `dstack-prototyping` skill now require Python serializers or quoted heredocs such as `<<'EOF'` for JSON/Markdown/YAML artifacts. -- [x] Backend-placement/task-first validation: `Qwen/Qwen2.5-7B-Instruct` on fleet `reusable-vs-container` had cheaper RunPod container offers and a JarvisLabs L4 reusable/inspectable offer. Claude chose JarvisLabs despite the slightly higher hourly price, submitted a task first, then promoted a final service that reused the warm L4 instance and verified `/v1/models` plus `/v1/chat/completions`. The endpoint was stopped and the temporary fleet deleted. This proves task-first on a reusable backend, but not the interactive SSH/dev-environment loop. -- [x] Probe-quality correction from the same validation: the task probe observed `nvidia-smi` and host driver evidence but failed on `python` missing before proving framework/runtime/server behavior. The final service still proved the endpoint, but the prompt and `dstack-prototyping` now state that host sanity checks are not recipe proof; a probe must exercise the intended serving stack before service promotion. -- [x] Probe-quality rerun exposed an earlier decision failure before it could test the probe fix: Claude saw RunPod and JarvisLabs offers but inferred "both backends are container-only (no reusable VM/SSH state)", skipped the task/dev probe, and submitted a direct RunPod service. We stopped the endpoint, terminated the service, and deleted the temporary fleet. Prompt/skill now forbid inferring "container-only" or "no reusable state" from offers alone and require uncertainty to be resolved, not treated as proof that a probe cannot help. -- [x] Second probe-quality rerun improved placement and run type but exposed a sharper harness miss: Claude chose JarvisLabs L4 and submitted a task first, but encoded the full investigation as one batch command chain and omitted all optional instance cache mounts (`configuration.volumes=[]`, `runtime.volume_names=[]`). We stopped the endpoint, terminated the task, deleted the fleet, and tightened prompt/skill so attach/SSH-capable probes should be long-lived interactive tasks/dev environments (`sleep infinity` or equivalent) and Hugging Face-style model-serving probes/services should use optional cache mounts or record why they were omitted. -- [x] Third probe-quality rerun showed the limit of prompt-only enforcement: Claude added the optional Hugging Face cache mount, but still generated a batch task instead of a long-lived attach/SSH probe, again wrote `jarvislabs+runpod (container-style)`, and submitted RunPod L4. We stopped the endpoint/run immediately and deleted the fleet. Next fix should be structural: server-generated backend/fleet capability context and possibly a pre-submit guard/required artifact for interactive-probe decisions, not more prose. -- [x] Backend docs-gated rerun validated the intended placement/prototyping path: `Qwen/Qwen2.5-0.5B-Instruct` on fleet `probe-quality-guard` classified JarvisLabs as VM-based from backend docs, rejected RunPod as container-based/no instance volumes, submitted a long-lived task with `sleep infinity`, attached/SSHed into it, verified vLLM locally, stopped the task, reused the idle JarvisLabs L4 for the service, verified `/v1/chat/completions` through the dstack service URL, reached `running`, and stopped cleanly. CLI UX now supports `dstack endpoint logs -w`, and foreground endpoint apply streams the endpoint progress log stream while polling status without `attach`. -- [x] Focused endpoint tests and static checks currently pass (`pytest` endpoint suites, `ruff`, `pyright`). - -### Still open for v1 - -- [ ] Harden the real Claude Code subprocess runtime based on live failures: multi-host restart behavior, duplicate-process prevention across server instances, packaging, and live-run efficiency. -- [ ] Finish runtime durability: final-report handoff after process/server crashes, multi-host reconciliation, and cleanup of non-final submitted runs after restart. -- [ ] Guard learned-preset quality: recipe service `resources` must stay broad enough for reuse but not too loose for correctness, exact hardware must remain in `validations`, and saved presets must be usable without the agent. -- [ ] Add server-generated backend/fleet capability context for the agent workspace: allowed fleets, current nodes/idle instances, backend capability facts known to dstack, whether attach/SSH/dev environments are expected to work, and whether instance cache/volumes are plausible. The last probe-quality run showed that asking the agent to infer this from docs/offers is unreliable. -- [ ] Add a lightweight pre-submit guard or required pre-submit artifact for create-recipe probes: when the chosen experiment is a task/dev probe and attach/SSH is available, the artifact must state whether it will be long-lived/interactive, whether cache mounts are included, and why batch commands are acceptable if used. -- [ ] Run a true interactive SSH/dev-environment scenario on reusable capacity. The JarvisLabs validations proved task-first and better placement, but not the intended long-lived attach/SSH loop, interactive command tuning, optional instance cache mounts, or stronger local server probing before promotion. -- [ ] Decide whether any lightweight preset metadata is worth adding later. Do not add final run ids, recipe sources, or verification summaries to the YAML until a concrete reader uses them. -- [ ] Runtime recipe grounding against vLLM recipes / SGLang docs / HF model cards, with mocked zero-network tests. -- [ ] Endpoint preset inspect polish: keep `dstack endpoint preset get --json` useful enough for exact `validations` and service recipe review without cluttering the compact list output. -- [ ] Real agent runtime dependencies installed automatically by normal server install/deploy paths: local `uv` server installs and server Docker images must include the runtime; no manual post-install dependency step. -- [ ] Endpoint update/version design before in-place updates: a `configuration_version`/deployment guard so stale background workers in multi-server Postgres setups cannot mark a newer endpoint config running. -- [ ] Documentation: endpoint reference schema page, env vars, concepts page, manual e2e runbook, example presets. - -### Critical assessment - -The endpoint storage/status/preset/run-link side is becoming reasonable. The feature is still not trustworthy because the agent loop and learned-preset quality are not yet proven across repeated real deployments. - -Highest current risks: - -- Agent works once but saves a preset recipe that is too exact, stale, missing validation evidence, or not reusable. -- Agent wastes budget because hardware selection, offer inspection, experiment choice, or failure recovery is weak. -- Agent logs/traces are either too noisy for users or too thin for debugging. -- Multi-host server restart during `prototyping` can leave ambiguous state or orphan submitted runs if the new server cannot observe the old host's Claude process/workspace. -- Multiple learned recipes for the same model accumulate without an explicit update policy. -- We overfit code around the first Qwen smoke runs before testing enough real model/backends/failure modes. -- One fresh Claude run used macOS-specific `sed -i` and read `~/.dstack/config.yml` to find URL/token for verification; both were prompt/context defects tightened afterward. -- A later fresh Claude rerun fixed those mechanics but failed the higher-level harness test: after service failures caused by FlashInfer runtime JIT, a RunPod provisioning stall, and CUDA 13 package/driver mismatch, the agent kept submitting services instead of switching to a better prototyping path when possible. It also exposed a stop bug: the endpoint became `stopped` while the Claude process continued and submitted another paid run. Process-group abort is now implemented and live-validated; the validation also fixed a zombie-only process-group edge case. -- The latest isolated no-fleet run passed mechanically and showed useful backend/fleet choice, but endpoint logs are still too sparse for watching a real investigation. The next improvement should make the agent write more natural-language decision updates without reintroducing hardcoded labels or templates. -- The latest isolated run also showed that fleet choice needs explicit evidence. `gpu:16GB:1`, `nodes: 0..1`, max price, and idle duration were plausible for a tiny Qwen model, but the harness must make the agent justify which existing fleet/resource envelope it uses without editing fleet configuration. -- Prompt instructions alone are not enough for the fleet/profile contract. The workspace-local CLI guard now enforces allowed fleets and common accepted profile constraints, but this must stay tied to real traces; if we accept more endpoint fields, they must either be enforced by the guard/submitted YAML or explicitly rejected/deferred. -- The 7B existing-fleet run succeeded, but also exposed real harness inefficiency: Claude spawned several concurrent shell polling loops instead of one bounded status loop. This is not just cosmetic; on long deployments it can waste process slots, tokens, and make command traces hard to read. -- Claude's first model API probe used a 420s internal deadline but was killed by Claude Code's 2-minute Bash tool timeout. The agent recovered by checking logs and retrying, but the harness should teach short timed probes and background polling explicitly instead of relying on recovery. -- The service proxy briefly returned `404 Service not found` immediately after service readiness. Claude retried and succeeded, so this is likely a proxy/registration timing edge, but the prompt/runbook should treat early 404s as retryable only after run JSON and service probes indicate the service is registered. -- The fresh Qwen2.5-1.5B RunPod run succeeded mechanically, but still showed prompt/harness gaps: Claude initially omitted the required `fleets` field and depended on the guard to catch it; it used a long polling command that hit the Claude Code Bash timeout; endpoint progress did not include the final "service running" and "model verified" milestones; and the handwritten `final_report.json` contained extra fields with malformed strings. The server ignored the extra fields and used the required schema fields, but future traces should be cleaner. -- The fresh RunPod run did not capture a direct `nvidia-smi` driver string. The service logs proved the cu129 vLLM image ran successfully with CUDA/FLASH_ATTN, which is useful evidence, but validation artifacts should distinguish "runtime worked" from "host driver version recorded." -- The fresh Qwen2-0.5B validation run showed that the latest prompt tightening helped materially, but it also exposed the opposite preset-resource risk. The saved preset is reusable, but because the final service YAML used `gpu: 1`, the recipe service resources had no GPU-memory floor even though validations prove RTX 2000 Ada 16GB. This is not necessarily wrong for a 0.5B model, but the harness needs a sharper rule for when count-only GPU constraints are acceptable versus when a memory envelope is required. -- The fresh Qwen2-0.5B validation run still used logs as part of readiness after an early proxy miss ("wait for Uvicorn running"). That was not fatal because the agent had already checked run JSON and later sent a real model request, but the intended loop is still: run JSON for lifecycle, model API probe for proof, logs for diagnosis/explanation. -- The next fresh Qwen2-0.5B run fixed the resource-envelope issue: the final service YAML used `gpu: 16GB..24GB:1` and the learned recipe kept exact RTX 3090 placement under `validations`. This is a good sign, but not enough to generalize to larger models or multi-replica deployments. -- The same run validated same-host server restart while Claude was running: the server restarted, reused the same agent session/workspace, and did not start a duplicate Claude process. Multi-host restart and Postgres multi-server coordination remain unproven. -- The same run also showed that the workspace CLI guard is necessary: Claude first used `backend` instead of `backends` and omitted `fleets`; the guard blocked preview before spend and Claude corrected the YAML. Treat guard catches as useful safety, but still reduce repeated prompt-level mistakes. -- Endpoint logs leaked Claude assistant stdout even though `progress.jsonl` was clean. The runtime now stops copying assistant stream text to endpoint logs; only progress messages and explicit server lifecycle messages belong there. -- A no-cost reuse preview after the same run selected an older Qwen2 recipe rather than the new resource-envelope recipe. The local store now merges duplicate files into one model-level preset. V1 keeps selection deterministic: try stored recipes in order and use the first one with available offers under the endpoint's effective fleets/profile constraints. Explicit user recipe selection stays Later. -- The 7B main-loop run is a better signal than the tiny Qwen runs: Claude chose a reasonable NVIDIA 24GB..48GB envelope, recovered from no-capacity, diagnosed a FlashInfer/nvcc failure, and produced a reusable recipe. It still showed that the harness should prefer pinned official vLLM serving images for final vLLM OpenAI services unless runtime package installation is itself the experiment. -- The probe-quality rerun showed that prompt-only backend capability reasoning is still fragile. The agent can overrule the intended reusable/inspectable placement route by inventing a backend-capability conclusion from an offer table. We either need stronger prompt/skill constraints or explicit backend/fleet capability context from the server before the agent decides task/dev vs service-first. - -### Immediate next steps: repeat the loop, then harden - -Do these in order. Do not add more harness surface area until the previous step has produced evidence. - -1. **Keep v1 recipe selection simple and deterministic.** Repeated agent tests now leave multiple provisionable recipes for the same model. Local duplicate files are merged. V1 should try recipes in stored order and use the first recipe whose normal run plan has available offers under the endpoint's effective fleets/profile constraints. Do not add quality ranking yet. -2. **Reduce harness waste exposed by guard catches.** The guard must stay, but the agent should not repeatedly write `backend` instead of `backends` or omit required `fleets` from final service YAML. -3. **Harden final-report handoff and submitted-run reconciliation.** Same-host restart while Claude is live now works, but the endpoint worker still needs stronger tests around final report written before link, process/server crash timing, and cleanup of non-final submitted runs. -4. **Deepen the reusable-capacity route.** The JarvisLabs run proved that the agent can prefer a reusable/inspectable backend over cheaper container offers and start with a task. Next, prove the stronger path: when attach/SSH/dev-environment is useful, the agent keeps an interactive probe alive, checks the intended image/runtime/command, and promotes only after the serving stack is actually exercised. Confirm hardware and expected GPU spend before each paid run. -5. **Keep agent budget/cost governance out of v1.** Add it later only with durable per-session accounting and an actually enforced runtime contract; do not expose a config field or env var before that exists. - -### How to overcome the current risks - -Treat this as a working hypothesis, not a fixed design. After each real endpoint run, update the plan with what actually happened and remove or change tactics that did not help. - -| issue | realistic mitigation | evidence to collect | reconsider if | -|---|---|---|---| -| Learned preset works once but is not reusable | Run an immediate preset-reuse test after every successful agent deployment. Validate that the saved recipe `service` can be planned/submitted without the agent and that `validations` are only evidence. | apply plan path selected, offers shown, final service run, endpoint status, model request result, saved preset YAML before/after | the preset path fails for reasons unrelated to transient capacity; then fix preset contract/builder before prompt changes | -| Recipe resources are too exact or too loose | Keep resources sourced from the final service config, but add validation/reporting that flags exact GPU names or full exact CPU/memory/disk constraints in learned recipes unless explicitly justified by the service shape. Do not auto-generalize blindly; first inspect the agent's service YAML and the run plan. | final service YAML, saved service resources, saved validations, offer count before/after, reason for any exact GPU/model constraint | exact constraints repeatedly block reuse, or broad constraints repeatedly schedule hardware that cannot serve the model | -| Agent chooses poor hardware/backend or wastes submissions | Give the agent a required decision trace: model sizing evidence, hardware envelope, current fleet-filtered dstack offers, backend/runtime characteristics inside allowed fleets, planned run resources, and why it is submitting now. Hourly price is a constraint, not the objective: cheaper container placement should not beat a slightly more expensive reusable placement if the reusable path reduces total iteration time, repeated downloads, image churn, or debugging risk. Start with prompt/context; only add server-side enforcement after observed bad behavior is clear. | recipe/model sources, offer table excerpt, chosen resources, rejected alternatives, selected backend placement, selected fleet, command count, elapsed time, GPU spend | the trace is absent or useless in two real runs, or the agent picks a container placement while a viable reusable placement existed without a good reason; then enforce a structured pre-submit decision artifact before paid `dstack apply` | -| Agent uses service submissions as every experiment | Make the prompt/skill require an explicit experiment-type choice. If the uncertainty is package/runtime, launch flags, model load, driver/CUDA compatibility, or backend provisioning and a viable VM/SSH/Kubernetes placement exists inside the allowed fleets, the agent should normally start with a task or interactive experiment before the final service. The agent still makes the final call; this is not a hardcoded rule. | run type per submitted run, why that type was chosen, elapsed provisioning time, whether reusable/inspectable placement was available, service vs task/dev command count, final proof | the agent keeps submitting services for non-URL-wiring uncertainty, or skips a useful task probe without a concrete reason; then add a structured pre-submit artifact that must justify service vs task/dev | -| Agent invents backend capability | Do not let the agent infer "container-only", "no reusable state", or "no SSH/dev value" from an offer table alone. It must use fleet state, backend/fleet behavior evidence, or mark capability as unknown and pick an experiment that resolves the unknown. If this repeats, pass explicit backend/fleet capability context from the server into the agent prompt. | fleet `nodes`/`idle_duration`, current idle/running instances, backend type evidence, agent reasoning line, chosen run type, whether task/dev was skipped | the next live run still misclassifies JarvisLabs/Lambda/Verda/Nebius/SSH/Kubernetes capacity; then make backend capability context server-generated instead of prompt-inferred | -| Task/dev probe is too shallow | Treat `nvidia-smi` and driver strings as host evidence only. A useful probe for service promotion should exercise the selected image or install path, Python/framework runtime, model/auth/cache access when feasible, server start on the intended port, and a local health or model API request if possible. If the probe exits before those checks, classify it as failed/inconclusive and either fix the probe or continue experimenting before promotion. | task/dev config, logs, driver evidence, framework import/version, model download result, local server/probe result, final service verification | two runs over-interpret partial probes; then require a structured pre-promotion artifact before the agent may submit the final service | -| Agent assumes global offers are usable | Enforce at the CLI boundary, not only in prompt text. The workspace-local `dstack` shim blocks fleet applies and blocks `dstack offer` / run applies that omit allowed fleets or violate accepted endpoint profile constraints. | fleet list/get output, fleet-filtered offers, backend/hardware reasoning, whether submitted configs carry the allowed fleet/profile constraints, cleanup, CLI guard block output | agent still finds a path around the guard, creates/edits a fleet, or submits a run outside allowed constraints | -| Host driver/CUDA compatibility is invisible in offers | Treat driver/runtime compatibility as uncertainty. Some backends vary host drivers by provider, GPU pool, region, or node, and dstack fleets/offers may not expose it. The agent should infer risk from provider/GPU/region evidence when possible and verify with `nvidia-smi`, run JSON/logs, dev environments, or one-shot tasks before trusting unpinned package installs. | `nvidia-smi`/driver evidence, package/image CUDA requirements, provider/GPU/region evidence, failed log root cause | driver mismatch recurs on the same provider; then bias recipes toward known-compatible images/package pins or avoid that offer class | -| Agent logs are noisy or unhelpful | Split logs into two layers: endpoint log gets natural-language decision updates authored by the agent; workspace trace stores command output, YAML, raw logs, and final report. Do not impose labels/categories/templates, and do not put full CLI output or service logs in endpoint logs. | `dstack endpoint logs ` readability, workspace artifacts, ability to diagnose failure without server stdout | endpoint logs still duplicate/replay, hide important state, force reading server logs for normal debugging, or stay too sparse to follow the investigation | -| Restart during `prototyping` duplicates work or loses the final run | Same endpoint configuration lifecycle reuses the same agent session/workspace; dead same-host Claude processes are relaunched in that workspace; new endpoint lifecycle creates the next session and passes older session summaries as read-only context. Still reconcile before launch: process metadata, final report, endpoint submission rows, linked `service_run_id`, and live submitted runs. | restart test with a live process, restart test after final report before link, endpoint submissions, linked run, no duplicate paid process | reconciliation logic becomes fragile, or multi-host restarts cannot observe enough process/workspace state; then consider moving agent execution to a separately tracked durable task model | -| Stop during `prototyping` leaks resources | Add process cancellation and submitted-run cleanup using `EndpointRunSubmissionModel`, not name heuristics. Endpoint stop should request abort, stop linked/submitted non-terminal runs, then finish through the normal run lifecycle. | stop during Claude process, stop after a submitted run appears, final run statuses, endpoint stopped, no running GPU run | cancellation depends on runtime behavior we cannot rely on; then isolate agent execution in a managed process/task with explicit supervisor state | -| Agent cost governance is not durable yet | Keep budget/cost accounting out of the v1 public endpoint schema. Add it later only when the runtime exposes reliable usage and dstack persists spend per endpoint agent session before any resume/retry starts another process. | Claude reported usage/cost, persisted session spend, refusal path when exhausted | Claude Code cannot expose reliable usage data; then use a stricter wall-clock/process cap plus operator-visible warning until runtime changes | -| Multiple learned recipes accumulate | Keep recipe selection simple: use the first stored recipe that has available offers under the endpoint's effective fleets/profile constraints. When the agent saves a newly verified recipe, store it before older recipes so immediate reuse sees the latest proof. Do not delete or rewrite older recipes until the harness can prove they are not reproducible under their claimed conditions. | recipe list by model, service resources, validation evidence, reuse success/failure per recipe, whether the freshly saved recipe is selected by a no-spend preview | users need control over a specific recipe; then add an explicit later `recipe: ` endpoint option before adding automatic ranking or update policy | -| Agent harness design overfits early Qwen runs | Maintain a scenario ladder: one tiny public model, one slightly larger common model, one gated/HF-token model, one no-offers/constraint case, one failure-cleanup case. Change harness only after comparing patterns across at least two scenarios unless the bug is clearly blocking. | per-scenario run notes, failure categories, command counts, spend, preset reuse result | the first two scenarios expose contradictory needs; then pause and rework the harness contract rather than patching locally | - -Near-term implementation strategy: - -1. Prefer prompt/context and artifact requirements for the first two real runs. They are cheaper to change and reveal what the agent naturally does. -2. Add server-side validation only where the failure would be expensive or dangerous: wrong project/user run, missing service `model`, missing final verification, leaked non-terminal runs, invalid preset shape. -3. Keep every hardening change tied to a reproducible trace: command transcript, endpoint log, final report, saved preset, and dstack run state. -4. After each real run, classify the issue as one of: dstack/backend provisioning bug, agent decision bug, prompt/context gap, preset contract bug, lifecycle/recovery bug, or UX/logging bug. Fix the right layer; do not let agent failures hide dstack/backend failures. -5. Be willing to change runtime approach if Claude Code cannot reliably run non-interactively, expose usage, cancel, preserve artifacts, or package cleanly in server installs/Docker. - -### Adjusted from the original plan - -- Preset service responsibility was narrowed to **storage only**. Matching, planning, and service apply live outside `EndpointPresetService`. -- Presets are now model-level files with one or more `recipes`. The recipe `service` is the scheduling source of truth; `validations` store exact running replica resources in service replica-group order. -- New learned GPU recipes should be vendor-aware because serving recipes are not portable across accelerator vendors. Existing vendorless local recipes are preserved on read; the agent harness is responsible for authoring vendor-aware final service YAML, and the learned-preset builder may fill the vendor from exact validation hardware before saving. -- Presets do **not** store backend, region, or instance type in v1. They store service resource requirements plus exact validation evidence that can be replanned against current fleets. -- A learned recipe records how many registered running replicas existed per replica group at save time through `validations[*].replicas[*].resources`. Autoscaling metrics, scaling validation, and benchmark metadata are Later. -- Preset matching does **not** force `creation_policy: reuse`; default `reuse-or-create` may use elastic fleets that can provision new instances. -- Endpoint plan `preset_policy` remains the configured/default policy; the selected path (`preset`, `agent`, or `none`) is represented only by `provisioning_plan`. -- Endpoint config changes are handled like current run UX: prompt to stop/override, with the backing service cleanup performed server-side. In-place update/rolling redeploy is Later. -- Later endpoint config updates must reuse the existing service-run `get_plan`/`apply_plan`/rolling deployment machinery, and endpoint DB updates must be guarded by a configuration/deployment version for multi-server safety. -- Endpoint-level `resources` are not part of v1. V1 placement constraints are the accepted `ProfileParams` plus the preset/service resources. If endpoint `resources` are added later, they must be a hard constraint with clear merge semantics for single-service and replica-group services; they must not be a prompt-only hint. -- Stop/override applies only to non-terminal existing endpoints. Terminal endpoints are replaceable like finished runs. -- Serving-run-name lookup is a conflict check for non-terminal runs only. Terminal conflicting runs follow normal run submission semantics: the run apply path can delete/recreate them, while endpoint code must not adopt/delete unrelated runs by name. -- No generic server-side endpoint health probe in v1. Existing service probes/registration are the server readiness source. In the agent path, the agent itself must perform final functional verification before reporting a final service. -- Automatic provisioning retry is Later. A failed endpoint is terminal for v1. -- Frontend UI is explicitly Later; no UI is required for v1. -- `DSTACK_AGENT_ANTHROPIC_API_KEY` remains the official server-agent env var. A local `DSTACK_ANTHROPIC_API_KEY` must be mapped/exported to that name if used. -- The raw `anthropic` Messages API loop is explicitly rejected for the real agent path. The next implementation must use a real agent runtime and package its dependencies automatically for server installs and Docker images. - ---- - -## 1. What we're building - -A new top-level dstack configuration type `endpoint`: the user declares *what model* they want served; dstack figures out *how* — either from a locally stored preset (one tested service deployment topology on ordered tested replica spec groups) that matches the project's existing fleets, or by launching an LLM agent (Anthropic) that authors and deploys a dstack service for that model. The endpoint follows the backing service lifecycle instead of adding a parallel serving layer. - -```yaml -type: endpoint -name: qwen3-32b # optional; auto-generated if omitted -model: Qwen/Qwen3-32B # required; HF model id / model name -env: - - HF_TOKEN # merged into whatever service gets deployed -# plus any ProfileParams: backends, regions, spot_policy, max_price, fleets, idle_duration, ... -``` - -``` -dstack apply -f endpoint.dstack.yml - └─ endpoint: SUBMITTED - └─ server background pipeline picks it up (multi-replica safe) - ├─ preset matches existing fleets? → submit service run from preset - ├─ else, agent enabled (DSTACK_AGENT_ANTHROPIC_API_KEY)? → agent deploys a service - └─ else → FAILED ("no matching preset found and server agent disabled") - └─ endpoint: PROTOTYPING (agent investigates/deploys, for the create-recipe path) - └─ endpoint: PROVISIONING (service run starting, model pulling, service probes passing) - └─ endpoint: RUNNING (service has a registered running job and model URL) -``` - -The endpoint's deliverable is an OpenAI-compatible base URL (from the backing service's `ServiceSpec.model.base_url`). - -### v1 scope (this plan) - -- New configuration type + DB model + REST API + CLI (`dstack apply`, `dstack endpoint list|get|logs|stop|preset`). -- `EndpointPipeline` background processing: SUBMITTED → PROTOTYPING/PROVISIONING → RUNNING/FAILED, STOPPING → STOPPED, crash recovery, and ownership-safe reconciliation. -- Preset subsystem: `EndpointPresetService` interface + local-directory implementation for storing/loading presets; endpoint planning code separately matches loaded presets against existing fleets, including elastic fleets that can provision new instances, via the run planner; successful agent deployments are saved back as sanitized local presets. -- Agent subsystem: `AgentService` interface + real Claude agent runtime implementation, a CLI-first execution workspace that lets the agent use the real `dstack` binary, vendored prompt/context, recipe/hardware grounding, and a minimal structured handoff for the final service. Raw LLM API loops are not accepted for v1. -- Endpoint readiness follows the backing dstack service lifecycle on the server side: a service run is usable once it is RUNNING, has a registered running job, and exposes a model URL. The agent path additionally requires the agent to verify the final service with a model request before reporting success; the server does not add a generic duplicate endpoint probe in v1. -- `DSTACK_AGENT_ANTHROPIC_API_KEY` + related settings; real agent runtime dependencies installed automatically by normal server install and Docker deployment paths. -- Endpoint apply/log UX: `dstack apply -f endpoint.dstack.yml` shows an advisory plan and confirmation first, then follows endpoint progress by polling server-side status/log storage; `dstack endpoint logs ` shows endpoint agent/progress logs, while backing service logs remain available via top-level `dstack logs `. No frontend UI in v1. - -Everything else → §12 "Later". - ---- - -## 2. Ground truth: existing machinery we build on - -These are the load-bearing facts a developer needs; each was verified in code. - -1. **Configuration registration hub** — `src/dstack/_internal/core/models/configurations.py`: `ApplyConfigurationType` enum, `AnyApplyConfiguration`, `BaseApplyConfiguration.__root__` (discriminated on `type`), `AnyDstackConfiguration` (feeds the editor JSON schema built in CI). CLI dispatch via `apply_configurators_mapping` in `src/dstack/_internal/cli/services/configurators/__init__.py`. There is **no generic server-side apply endpoint** — every resource type has its own typed router + `APIClient` group. -2. **Non-run resource blueprint** — gateways and volumes show the durable pipeline pattern: model with `PipelineModelMixin` (`lock_expires_at`/`lock_token`/`lock_owner`, `models.py:204`) + `status`/`status_message`/`last_processed_at`; status enum in core models; service module (`services/volumes.py` is the cleanest); `Pipeline` subclass registered in `PipelineManager.__init__` (`background/pipeline_tasks/__init__.py:35-48`); router + schemas; configurator. Gateways/volumes also use an async delete flag, but endpoints intentionally do not expose delete in v1; endpoint lifecycle is stop/history. -3. **Multi-replica safety** is NOT the in-memory locker (`services/locking.py` — no-op `DummyResourceLocker` on Postgres by design). It is: durable lock columns claimed in the fetch transaction with `SELECT ... FOR UPDATE SKIP LOCKED`, a `Heartbeater` that checks tracked items every ~1s and extends `lock_expires_at` whenever a lease comes within `heartbeat_trigger` of expiry (so with 30s timeout / 15s trigger, ~every 15s), and every subsequent write guarded by `WHERE id = :id AND lock_token = :token`. If a replica dies, the lease expires (≤ ~30s) and another replica's fetcher re-claims the row (`or_(lock_expires_at.is_(None), lock_expires_at < now)`, ordered by `last_processed_at ASC`). Long operations are legitimate: the heartbeater extends the lease indefinitely (`instances/cloud_provisioning.py` runs minutes-long provisioning inside one `process()` call), but crash recovery restarts the whole step — so steps must be reconcile-first/idempotent. -4. **Server-side run creation is first-class** — `src/dstack/_internal/server/services/runs/__init__.py`: `submit_run(session, user, project, run_spec, pipeline_hinter=None) -> Run`, `stop_runs(...)`, `delete_runs(...)`, `get_run_by_name(...)`, `get_plan(session, project, user, run_spec, max_offers)`. Repo-less runs work: leave `run_spec.repo_id`/`repo_data` as `None` → `validate_run_spec_and_set_defaults` (`runs/spec.py`) fills the virtual repo (`DEFAULT_VIRTUAL_REPO_ID = "none"`) and `_get_run_repo_or_error` upserts the `RepoModel` row. Requires `run_spec.ssh_key_pub` or `user.ssh_public_key` set — and note `get_plan` calls the same validation, so this bites at *planning* time too. `submit_run` **commits the session multiple times** — always call it (and `stop_runs`/`delete_runs`) with a fresh `get_session_ctx()`, never inside a pipeline worker's token-guarded transaction. -5. **Run teardown is asynchronous and two-phase.** `stop_runs` only sets `termination_reason` + status `TERMINATING`; the RunPipeline performs actual termination. `TERMINATING` is **not** in `RunStatus.finished_statuses()` (`[TERMINATED, FAILED, DONE]`), and `delete_runs` raises `ServerClientError("Cannot delete active runs: ...")` for any non-finished run. Neither function raises for missing names — they filter and silently no-op. **Any "stop then delete" flow must wait (across pipeline iterations) for the run to finish before deleting.** -6. **Service readiness ≠ `RunStatus.RUNNING`.** A run is RUNNING when *any* replica job is RUNNING (`pipeline_tasks/runs/active.py`). The service-level readiness signal is `JobModel.registered`, set `True` only after all service probes pass (`_maybe_register_replica`, `jobs_running.py:1158`; it is set back to `False` only in the terminating pipeline, `jobs_terminating.py:787`). If `model:` is set and `probes` omitted, a default `POST {prefix}/chat/completions` probe is auto-generated (`services/jobs/configurators/base.py::_openai_model_probe_spec`). Endpoint v1 relies on this existing service mechanism instead of adding a parallel endpoint probe loop. -7. **In-process model probing exists but is not generic endpoint v1 behavior** — `_execute_probe` (`background/scheduled_tasks/probes.py:106-126`) POSTs `chat/completions` to a replica through `get_service_replica_client(job_model)` (`services/jobs/job_replica_http_client.py`, SSH tunnel + httpx over a Unix socket; URL host is a dummy `http://dstack`; the whole call is wrapped in `except (SSHError, httpx.RequestError) → False`). This is useful background for later agent verification/hardening, but v1 endpoint readiness should not duplicate service probes. -8. **Preset↔fleet matching primitive** — `_get_job_plan` (`services/runs/plan.py:970-988`): plan offers always include existing-instance offers, and add new-cloud offers only when `profile.creation_policy == CreationPolicy.REUSE_OR_CREATE` **and** `profile.instances is None`. This is exactly the "existing fleets" semantics we need: a non-empty available offer can mean either an already-idle instance or an elastic fleet that can create a new matching instance. If the user explicitly sets `creation_policy: reuse`, matching is restricted to currently existing instances; otherwise the default `reuse-or-create` allows elastic fleet capacity. (Caveats: each candidate evaluation can take seconds because the planner enumerates offers; and the per-job criterion under-checks total capacity for multi-replica configs — see §7.3.) -9. **No system user exists.** Only `admin` is guaranteed (`services/users.py::get_or_create_admin_user`, created at startup). Runs are only ever **soft-deleted** (`delete_runs` sets `RunModel.deleted=True`; nothing hard-deletes run rows except the project-cascade) — an `ondelete=SET NULL` on a run FK effectively never fires; handlers must treat `run.deleted == True` as "run gone". `RunModel` has no tags column; for endpoints, the serving run uses an explicit FK (`service_run_id`) and endpoint-submitted run history lives in `EndpointRunSubmissionModel`. Name/user/timestamp checks are not a durable ownership model. -10. **Packaging baseline**: hatchling normally packages `src/dstack` plus declared artifacts; repo-root `skills/` must be force-included explicitly or an installed server will not have them. This branch force-includes `skills/**` for wheels/sdists and keeps the endpoint-specific system prompt under `src/dstack`. **dstack pins `pydantic>=1.10.10,<2.0.0`** — do not import an agent runtime that requires pydantic v2 into the server process (§8.2). -11. **Agent skills live under repo-root `skills/`** — `skills/dstack/SKILL.md` and `skills/dstack-prototyping/SKILL.md` are the canonical skill files. `pyproject.toml` packages `skills/**` into wheels/sdists, and the endpoint harness locates that packaged `skills` directory before copying the required skills into each agent workspace's `.claude/skills`. -12. **A "templates" feature already exists** (`core/models/templates.py::UITemplate`, `services/templates.py`, `DSTACK_SERVER_TEMPLATES_REPO`) — UI-only, git-repo-sourced run-config templates. Presets must not collide with this namespace (we use "preset" consistently); reusing its git-repo fetch machinery is a Later option. -13. **External recipes**: vLLM recipes expose a machine-readable JSON API built for agents — `https://recipes.vllm.ai/models.json` (index, ~500 entries, dedupe on `derived_from`) + `https://recipes.vllm.ai//.json` (exact `vllm serve` command, docker image, per-GPU configs). SGLang Cookbook lives at `docs.sglang.io/cookbook` (source: `sgl-project/sglang:docs_new/cookbook`, MDX; discoverable via `https://docs.sglang.io/llms.txt`; pages fetchable raw by appending `.md`, but some embed commands in JSX). Both Apache-2.0 — runtime fetching and vendoring with attribution are fine. The old `sgl-cookbook` repo is archived read-only. -14. **Research notes behind this plan are now stored in-repo** under `endpoint-implementation-research/`, copied from `/private/tmp/claude-501/-Users-dstack-dstack/b621b4a9-b6ee-4b6b-813e-1084018def84/scratchpad/research/` plus `verify-findings.md`. These are planning artifacts, not packaged runtime resources; the packaged endpoint prompt and skills live under `src/dstack/_internal/server/services/endpoints/agent/` (§8.4). - ---- - -## 3. Configuration & core models - -New file `src/dstack/_internal/core/models/endpoints.py`: - -```python -class EndpointStatus(str, Enum): - SUBMITTED = "submitted" - PROVISIONING = "provisioning" - PROTOTYPING = "prototyping" # server agent is investigating/deploying - RUNNING = "running" # backing service is ready - STOPPING = "stopping" - STOPPED = "stopped" - FAILED = "failed" - -class EndpointConfiguration(ProfileParams, generate_dual_core_model(EndpointConfigurationConfig)): - type: Literal["endpoint"] = "endpoint" - name: Optional[str] = None # dstack resource name; auto-generated if omitted - model: str # required; HF model id / served model name - env: Env = Env() - -class Endpoint(CoreModel): # API/runtime representation - id: uuid.UUID - name: str - project_name: str - user: str - configuration: EndpointConfiguration - created_at: datetime - status: EndpointStatus - status_message: Optional[str] - run_name: Optional[str] # backing service run (if provisioned) - url: Optional[str] # OpenAI-compatible base URL from ServiceSpec.model.base_url (read-time derived) - error: Optional[str] -``` - -Notes: -- `ProfileParams` (`core/models/profiles.py:310-493`) gives us `backends/regions/spot_policy/max_price/creation_policy/idle_duration/fleets/tags/...` for free — this is exactly the "profile params" requirement. Follow the run-config MRO pattern (`ProfileParams` first) and the pydantic-duality rule: never define `class Config` directly; create `EndpointConfigurationConfig(ProfileParamsConfig)` chaining `schema_extra` and pass it via `generate_dual_core_model(...)` as the last base (see `ServiceConfigurationConfig`, `configurations.py:1316`). -- Codebase is **pydantic v1** (`validator`/`root_validator`, `.json()`/`.parse_raw()`); do not write v2 APIs. -- No `EndpointSpec` type in v1 — `CreateEndpointRequest` takes a bare `configuration` (volumes parity, `CreateVolumeRequest`); introduce a spec wrapper together with the plugins work (§12) if/when needed. -- `Env` is a plain BaseModel with a custom root (`core/models/envs.py`) — bare `VAR` entries are `EnvSentinel`s resolved from `os.environ` **CLI-side only** (`ApplyEnvVarsConfiguratorMixin.apply_env_vars`). The server must reject configurations arriving with unresolved sentinels (validate in `create_endpoint`; `Env.as_dict()` raises `ValueError` on unresolved). -- Validate `name` with `validate_dstack_resource_name` (regex `^[a-z][a-z0-9-]{1,40}$`, `core/services/__init__.py`). `model` is deliberately a plain `str` in v1 (not `AnyModel`) — it's a requirement, not a service config field. - -**Registration checklist** (all in `core/models/configurations.py` unless noted): -1. Add `ENDPOINT = "endpoint"` to `ApplyConfigurationType` (~:1384). -2. Add `EndpointConfiguration` to `AnyApplyConfiguration` (~:1393). -3. Add it to `BaseApplyConfiguration.__root__` union (~:1401) — it's a "final configuration" (single `type` discriminator; no second-stage parse like volumes). -4. Add it to `AnyDstackConfiguration` (~:1440) — the editor JSON schema in CI (`.github/workflows/build-artifacts.yml`, `DstackConfiguration.schema_json()`) picks it up automatically. -5. Watch for circular imports: `configurations.py` already imports fleets/gateways/volumes models the same way. - ---- - -## 4. DB model, migration, events - -`src/dstack/_internal/server/models.py` — follow the normal pipeline model shape, but do not copy volume soft-delete semantics: endpoints have stop/stopped, not delete. - -```python -class EndpointModel(PipelineModelMixin, BaseModel): - __tablename__ = "endpoints" - - id: UUIDType(binary=False) pk, default=uuid.uuid4 - name: Mapped[str] = mapped_column(String(100)) - project_id: FK("projects.id", ondelete="CASCADE") + relationship - user_id: FK("users.id", ondelete="CASCADE") + relationship # endpoint creator; runs are submitted as this user - service_run_id: Mapped[Optional[uuid.UUID]] = FK("runs.id") + relationship - """The service run currently backing this endpoint (authoritative link; run name is convention only). - Runs are soft-deleted, so handlers must treat run.deleted == True as 'run gone' — no ondelete magic applies.""" - configuration: Mapped[str] = mapped_column(Text) # EndpointConfiguration JSON - status: Mapped[EndpointStatus] = mapped_column(EnumAsString(EndpointStatus, 100), index=True) - """Must be changed only via switch_endpoint_status() (API side) / token-guarded pipeline updates.""" - status_message: Mapped[Optional[str]] = mapped_column(Text) - provisioning_method: Mapped[Optional[str]] = mapped_column(String(100)) # "preset:" | "agent"; NULL until dispatched - created_at: NaiveDateTime, default=get_current_datetime - last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime) # MUST equal created_at on insert (fetch fast-path) - - __table_args__ = ( - UniqueConstraint("project_id", "name", name="uq_endpoints_project_id_name"), - Index("ix_endpoints_pipeline_fetch_q", last_processed_at.asc()), - ) -``` - -- One row per `(project_id, name)`. There is no endpoint soft delete in v1. Terminal same-name reapply resets the existing row and `EndpointRunSubmissionModel` preserves backing run submission history. -- JSON goes in `Text` columns as pydantic JSON — there is no JSON column type in this codebase. -- **Migration**: one new table, one migration. Generate with `cd src/dstack/_internal/server && alembic revision -m "add endpoints" --autogenerate`; lands under `migrations/versions/2026/` (per `alembic.ini` `file_template`). Plain `op.create_table` (a new table is additive ⇒ zero-downtime-safe; `contributing/MIGRATIONS.md`'s "separate migrations per table" rule concerns ALTERs of existing tables, and its `CREATE INDEX CONCURRENTLY` guidance applies to existing tables only). Column types: `sqlalchemy_utils.types.uuid.UUIDType(binary=False)`, `dstack._internal.server.models.NaiveDateTime()`. Remember tests create schema via `metadata.create_all`, so a *missing* migration won't fail unit tests — the migration is its own review item. -- **Events**: add `ENDPOINT` to `EventTargetType` (`core/models/events.py:12`) and an `EndpointModel` branch to `events.Target.from_model` (`server/services/events.py:89`) — `from_model` raises `ValueError` for unregistered types, so forgetting this crashes the first `emit()`. - ---- - -## 5. Server service module, REST API, CLI - -### 5.1 Service module — `src/dstack/_internal/server/services/endpoints/` (package: `__init__.py`, `presets.py`, `planning.py`, `agent/`) - -- `create_endpoint(session, project, user, configuration, pipeline_hinter) -> Endpoint`: - validate name + reject unresolved `EnvSentinel`s → name-uniqueness critical section (`lock_namespace = f"endpoint_names_{project.name}"`; SQLite: commit first + in-process lockset; Postgres: `pg_advisory_xact_lock(string_to_lock_id(ns))`) → duplicate non-terminal name ⇒ `ResourceExistsError` → duplicate terminal name ⇒ reset the same row to `SUBMITTED` with the new config, clear `service_run_id`/status message/provisioning method/lock fields, and refresh `created_at == last_processed_at == now` → auto-name via `generate_name()` loop if unset → insert row with `status=SUBMITTED`, `created_at == last_processed_at == now` → emit create/submit event → commit → `pipeline_hinter.hint_fetch(EndpointModel.__name__)`. -- `stop_endpoints(session, project, names, user)`: set `status=STOPPING` for non-finished endpoints, emit a stop event, commit, and `hint_fetch`. The pipeline owns stopping the linked backing service run and marking the endpoint `STOPPED`. -- `list_endpoints` (keyset pagination on `(created_at, id)` like volumes), `get_endpoint_by_name`, `endpoint_model_to_endpoint` (parse config via `EndpointConfiguration.__response__.parse_raw`; populate `run_name`/`url` at read time from the joined run's `service_spec` when present), `switch_endpoint_status(session, endpoint_model, new_status, actor=SystemActor())` + `emit_endpoint_status_change_event`. -- v1 does **not** call `apply_plugin_policies` — `ApplySpec` TypeVar (`dstack/plugins/_models.py:8`) doesn't include endpoints; extending it is a Later item (do not call the function with an unsupported type). - -### 5.2 REST API - -- `src/dstack/_internal/server/schemas/endpoints.py`: `ListEndpointsRequest`, `GetEndpointRequest{name}`, `GetEndpointPlanRequest{configuration, configuration_path?}`, `CreateEndpointRequest{configuration: EndpointConfiguration}`, `StopEndpointsRequest{names}` (all `CoreModel`). -- Endpoint configuration includes `preset_policy` (`reuse`, `create`, `reuse-or-create`, default `reuse-or-create`). This is distinct from profile `creation_policy`: `preset_policy` chooses whether endpoint provisioning may reuse a tested service recipe or ask the server agent to create one; `creation_policy` still controls whether the resulting service can reuse existing instances or provision new ones from elastic fleets. -- New core model `EndpointPlan`: `project_name`, `user`, `configuration`, `configuration_path`, `current_resource`, `action`, `preset_policy`, and `provisioning_plan`. `preset_policy` is the configured/default policy shown in the CLI; the selected path (`preset`, `agent`, or `none`) belongs in `provisioning_plan`. -- `provisioning_plan` is a small discriminated union: - - `type="preset"`: a **provisionable preset recipe** was selected. This includes `preset_model`, `recipe_id`, `service_name`, and `job_offers: list[EndpointPlanJobOffers]` derived from `runs.get_plan` (enough for the CLI to print the selected preset/recipe, stable run-like scheduling properties, and first matching offers, without dumping the full run plan). - - `type="agent"`: no provisionable preset was found and the policy allows creation. The plan may include a short reason such as "matching preset qwen has no available offers"; the initial plan must not invent final offers because the agent may explore hardware and service shapes within endpoint/profile constraints. - - `type="none"`: no provisionable path is available, with the exact failure message that the pipeline will use (for example, `preset_policy: reuse` with no matching preset or only no-offer presets, or `preset_policy: reuse-or-create` with no provisionable preset and no usable server agent). -- Preset terminology: - - A **model-matched preset** has the requested model and can be parsed/planned. - - A **provisionable preset** is a model-matched preset whose run plan has at least one available offer for every job plan. - - Only provisionable presets are submitted automatically. A no-offer preset is shown as context; if `preset_policy: reuse-or-create` and the agent is available, the pipeline falls through to the agent path instead of submitting the doomed preset. With `preset_policy: reuse`, no-offer presets stop with the normal no-offers/no-fleets UX. -- Plan semantics are advisory. `/get_plan` runs the same preset matching code and agent-enabled check that the pipeline will run, but the pipeline re-evaluates after submit because fleets/presets/agent availability may change. This mirrors run apply's UX without introducing endpoint in-place update in v1. -- `src/dstack/_internal/server/routers/endpoints.py`: `project_router = APIRouter(prefix="/api/project/{project_name}/endpoints", tags=["endpoints"], responses=get_base_api_additional_responses())` with `/list`, `/get`, `/get_plan`, `/create`, `/stop`. **Permission: `ProjectMember()`** (volumes precedent — endpoints are project workloads like runs, not admin infra like gateways). `/create` and `/stop` take `pipeline_hinter: Annotated[PipelineHinterProtocol, Depends(get_pipeline_hinter)]`. Responses via `CustomORJSONResponse`. -- Register in `server/app.py::register_routes` next to volumes (~:257). -- API client: `src/dstack/api/server/_endpoints.py` `EndpointsAPIClient(APIClientGroup)` (`list/get/get_plan/create/stop`, parse with `parse_obj_as(Endpoint.__response__, ...)`) + `endpoints` property on `APIClient` (`api/server/__init__.py`). -- No endpoint `/apply_plan` in v1: submit remains `/create`, and changed existing endpoints are handled by CLI-requested stop/override. The CLI requests endpoint stop and polls until the endpoint is terminal; the next `/create` resets the same endpoint row to `SUBMITTED` with the new config. In-place update/rolling redeploy is Later. - -### 5.3 CLI - -- `src/dstack/_internal/cli/services/configurators/endpoint.py`: `EndpointConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator)` with `TYPE = ApplyConfigurationType.ENDPOINT`. Model on `VolumeConfigurator`: - - `apply_configuration`: resolve env sentinels (`apply_env_vars`), call `api.client.endpoints.get_plan(...)`, print the endpoint plan, confirm, then `api.client.endpoints.create(...)`; on name collision: if the existing endpoint has the same config, print no changes and exit unless `--force`; if the config changed, show that the endpoint and its backing service will be stopped and replaced, then require confirmation (or `-y`) before calling `endpoints.stop`, polling until the endpoint becomes terminal, and creating a fresh endpoint. The CLI must not stop backing runs directly. This mirrors run apply's "stop and override" v1 behavior, not an in-place update; in-place update/rolling redeploy is Later (§12). - - Plan output: keep it close to run apply. Print one stable properties table: project, user, configuration path, type, spot policy, max price, preset policy, and preset model/recipe only when a provisionable recipe was selected. Do not show model, action, endpoint name as a separate row, resources as a separate property, backing service name, agent internals, or a separate "Provisioning" section. If a provisionable recipe matched, print offers underneath using the run-style offers table; that offers table is where the concrete resource information belongs. If only no-offer recipes matched, print one short message below the table; with `reuse-or-create` and an available agent, continue to the agent path. - - TODO: For `preset_policy: create` or agent fallback, `dstack apply` may later show initial candidate offers the agent can consider. These offers are advisory only: they must not be presented as selected endpoint resources or final hardware because the agent may still choose a different working service shape after investigation. - - Non-detached apply should **imitate attached progress without using attach**: poll `endpoints.get` for status and stream the endpoint's own progress log stream. Do not reserve local ports, open SSH tunnels, or call `run.attach`. Backing service stdout/stderr remains available through `dstack logs `. - - Detached apply (`-d`) submits and exits with the normal `submitted, detaching...` message. Follow-up commands (`dstack endpoint list`, `dstack endpoint get --json`, and `dstack endpoint logs `) are documented/help text, not extra foreground hints. - - `delete_configuration`: unsupported for endpoints. `dstack delete -f endpoint.dstack.yml` should fail clearly and point to `dstack endpoint stop `. - - Provisioning can take tens of minutes (model pulls, agent runs), so `-d` should work exactly as with runs. Keep foreground output concise; document follow-up commands in help/docs. -- Register in `apply_configurators_mapping` (`cli/services/configurators/__init__.py:27-49`). `dstack apply` and `dstack apply -h endpoint` then work with no further CLI edits; `dstack delete -f endpoint.dstack.yml` is explicitly rejected by `EndpointConfigurator.delete_configuration`. -- `src/dstack/_internal/cli/commands/endpoint.py`: `EndpointCommand` (`dstack endpoint list` / `get --json` / `logs` / `stop` / `preset list|get --json|delete`); register in `cli/main.py`. -- Nice property to document: preset-backed endpoints use a readable endpoint-derived service run name (normally `-serving`). Agent-backed endpoints may use a different concise run name chosen during investigation; once linked, backing service logs are available by the linked run name. - -### 5.4 Endpoint logs - -Support endpoint logs in v1, but only via detached/server-side log polling: - -- Add `dstack endpoint logs `. Top-level `dstack logs` remains run-only so same-name endpoint/run collisions do not silently show the wrong stream. Endpoint logs are concise agent/pipeline progress events, not backing service stdout/stderr. -- Do not introduce `dstack attach` for endpoints in v1. Endpoint apply/logs must use configured log storage (`FileLogStorage`, CloudWatch, GCP Logging, Fluent Bit/Elasticsearch, etc.) via the existing server log service. Backing service logs remain normal run logs and are requested with the service run name. -- The agent should emit major progress events to the endpoint log stream in real time. Detailed command output/transcripts stay in the endpoint workspace/debug trace so server logs and endpoint logs do not fill with huge YAML or CLI output. -- Tests: `dstack endpoint logs` reads the endpoint stream and does not fall through to the backing service run; top-level `dstack logs` remains run-only; non-detached endpoint apply does not call `run.attach`. - ---- - -## 6. Background processing: `EndpointPipeline` - -New file `src/dstack/_internal/server/background/pipeline_tasks/endpoints.py`, cloned from `volumes.py` (the cleanest single-model pipeline), registered in `PipelineManager.__init__` (`background/pipeline_tasks/__init__.py:35-48`). - -### 6.1 Pipeline wiring - -- `EndpointPipelineItem(PipelineItem)` + `{status}`; `EndpointPipeline(Pipeline[EndpointPipelineItem])`, `hint_fetch_model_name -> EndpointModel.__name__`. -- Tuning: `workers_num=4` (each in-flight agent run occupies a worker for minutes — this is the per-replica concurrency bound for paid agent sessions), `min_processing_interval=10s`, `lock_timeout=30s`, `heartbeat_trigger=15s`. The `Heartbeater` extends the lease indefinitely during a long `process()` call (precedent: `instances/cloud_provisioning.py`), so minutes-long agent steps are safe while the replica lives. -- `EndpointFetcher.fetch` — copy the volume fetch query shape, with the endpoint status filter and a longer effective interval for RUNNING rows (precedent: `InstanceFetcher`, `instances/__init__.py:190-211`, uses `min_processing_interval * 2` for steady-state statuses; we use a larger multiplier via an explicit condition): - - `WHERE status IN (SUBMITTED, PROTOTYPING, PROVISIONING, STOPPING) OR (status == RUNNING AND last_processed_at <= now - ENDPOINT_RUNNING_CHECK_INTERVAL)` - - `AND (last_processed_at <= now - min_processing_interval OR last_processed_at == created_at)` - - `AND (lock_expires_at IS NULL OR lock_expires_at < now)` - - `AND (lock_owner IS NULL OR lock_owner == EndpointPipeline.__name__)` - - `ORDER BY last_processed_at ASC LIMIT :limit FOR UPDATE SKIP LOCKED (key_share)` - - claim: one `lock_token` per batch, `lock_expires_at = now + lock_timeout`, `lock_owner = EndpointPipeline.__name__`; commit. -- `EndpointWorker.process`: refetch by `(id, lock_token)` with joinedloads — **must include `EndpointModel.project → ProjectModel.backends`** (in addition to `user` and `service_run`): the preset-matching path reaches `get_project_backends(project)`, which iterates the lazy `project.backends` relationship and raises `MissingGreenlet` under async SQLAlchemy if not eager-loaded (established pattern: `background/pipeline_tasks/volumes.py:237`, `gateways.py:232`). On token mismatch `log_lock_token_mismatch`; dispatch by state; build `_EndpointUpdateMap(ItemUpdateMap)` (adds `status`, `status_message`, `service_run_id`, `provisioning_method`); final write via `update(EndpointModel).where(id==..., lock_token==...).values(**update_map).returning(id)` after `set_processed_update_map_fields` + `set_unlock_update_map_fields` + `resolve_now_placeholders`; 0 rows ⇒ `log_lock_token_changed_after_processing` and abandon. Emit status-change events in the same transaction. Instrument with `@sentry_utils.instrument_pipeline_task(...)`. - -**Transaction rule (critical):** run submission/termination (`submit_run`, `stop_runs`, `delete_runs`) commit their own sessions and take their own advisory locks. The worker must call them through a **fresh `get_session_ctx()`**, never inside its own fetch/update session. Interleaving is safe because the endpoint row itself stays lease-locked. - -**Interim token-guarded updates:** if the agent path needs to persist progress before a long agent call, use an interim `UPDATE ... WHERE id AND lock_token` (+ commit). This is mechanically consistent with the framework (it leaves the lock columns untouched, so the heartbeater and the final write are unaffected; precedent for mid-process commits: related-row lock claims in `fleets.py` / `jobs_submitted.py`) — but the handler must then make sure the final update map doesn't overwrite interim values with stale ones. - -### 6.2 State machine - -``` -SUBMITTED ──(preset selected; deterministic service run name is taken)──► FAILED "run name '' is taken by an existing run" -SUBMITTED ──(preset matched; run submitted)───────► PROVISIONING (method=preset:) -SUBMITTED ──(no preset, agent on)─────────────────► PROTOTYPING (method=agent) -SUBMITTED ──(preset_policy=reuse; no preset)──────► FAILED "No matching endpoint presets found." -SUBMITTED ──(effective policy=create; agent off)──► FAILED "No matching endpoint presets found. Preset - policy create requires the server agent, - but DSTACK_AGENT_ANTHROPIC_API_KEY is not set." -PROTOTYPING ─(agent reports verified service run; run not ready yet)──► PROTOTYPING (service_run_id linked) -PROTOTYPING ─(linked run RUNNING + replica registered + model URL)────► RUNNING -PROVISIONING ─(preset run RUNNING + replica registered + model URL)► RUNNING -PROVISIONING ─(run finished/soft-deleted)──────────────────────────► FAILED -RUNNING ─(run gone/failed/deleted)─────────────────────────────────► FAILED (stop backing run) # later: agent retry policy -any ─(stop requested)──────────────────────────────────────────────► STOPPING → stop linked backing run → STOPPED -``` - -V1 does **not** automatically retry failed provisioning attempts. A backing run that finishes or fails before readiness makes the endpoint FAILED. - -#### `_process_submitted_endpoint` -1. Parse `EndpointConfiguration`; ask endpoint planning code (`planning.py::find_matching_preset_plan(...)`) to list presets through `EndpointPresetService`, build candidate run specs, and call the run planner (§7). -2. If a provisionable preset recipe matches: check the run name in that selected plan. A non-terminal run conflict that is not the linked `service_run_id` fails with a clear message; terminal conflicts are left to normal run submission, which can recycle them like service apply. Then submit the matching plan in a fresh session as the endpoint's creator user through a small internal helper that mirrors the run router (`refresh_ssh_key` if needed, `get_plan`, then `apply_plan`/`submit_run` with the same policy-applied spec), and return `{status: PROVISIONING, provisioning_method: f"preset:{preset.model}#{recipe.id}", service_run_id: run.id}`. `register_service` runs synchronously during submission and can raise (`FORBID_SERVICES_WITHOUT_GATEWAY`, referenced gateway missing) — catch `ServerClientError` ⇒ FAILED with the message. -3. Else if agent enabled (`settings.AGENT_ANTHROPIC_API_KEY` set and the real agent runtime is available): return `{status: PROTOTYPING, provisioning_method: "agent"}` — the agent runs in the PROTOTYPING handler (keeps SUBMITTED processing fast and makes long agent work visible in CLI/API). Do not block this path on the preset path's deterministic service-run-name convention; the agent uses strict numbered submission names, and the server links the final run by ID after validating project/user/config. -4. Else FAILED (message above; if the key is set but the runtime is unavailable, say the server agent implementation/runtime is unavailable; normal server installs and Docker images must include runtime dependencies once the implementation lands). - -#### `_process_provisioning_endpoint` — reconcile-first -1. **If `service_run_id` set**, load the run (treating `run.deleted == True` as "run gone"): - - run `RUNNING`, ≥1 replica job with `registered == True`, and `ServiceSpec.model.base_url` exists → if `provisioning_method == "agent"`, save a sanitized preset from the backing run (§7.4), then `RUNNING`. This deliberately relies on the existing service probe/registration path rather than adding a second endpoint probe. - - run in `finished_statuses()` or soft-deleted → FAILED with the run error/status. Automatic retry is Later. - - run `TERMINATING` → no-op (wait). - - run still starting (SUBMITTED/PROVISIONING/PULLING) → no-op; stay PROVISIONING. -3. **PROTOTYPING / no run yet + method=agent** — the long step: - a. Before launching or relaunching the agent, reconcile the endpoint workspace, existing final-report artifacts, endpoint submission rows, and live submitted/final runs. Same endpoint configuration lifecycle reuses the same agent session/workspace; if the same-host Claude process is gone without a terminal report, the server relaunches Claude in that workspace so it can resume from artifacts and submitted-run evidence. A new endpoint lifecycle starts the next session and gives the agent `previous_sessions.md` as read-only context. Cleanup of endpoint-submitted runs must use explicit `EndpointRunSubmissionModel` rows, never name matching. - b. Run `AgentService.provision_endpoint(...)` (§8) inside the worker — heartbeater keeps the lease alive while the agent subprocess runs. Stop-time cancellation exists for same-host Claude processes; multi-host restart/supervision remains a hardening item. Do not add a generic endpoint provisioning timeout here; real agent-session budgets belong inside the harness once token/cost/spend semantics are defined. - c. On structured success `{run_id}`/linked service run → set `service_run_id` and keep status `PROTOTYPING` until the linked service run also satisfies normal dstack service readiness. Once ready, save the learned preset and mark the endpoint `RUNNING`. On failure/abort → FAILED with the agent's summary in `status_message` + stop only the linked service run; cleanup of non-final submitted runs must use explicit endpoint submission records, not name matching. -4. **No run yet + method=preset**: reachable only if the preset run vanished before the endpoint recorded its linked service run — FAILED. Automatic retry is Later. - -#### `_process_running_endpoint` -Runs at the slower RUNNING cadence (`ENDPOINT_RUNNING_CHECK_INTERVAL`, default 60s): -1. Run liveness: backing run missing/soft-deleted/`finished_statuses()`/`TERMINATING` ⇒ FAILED "Backing service run is " (v1; re-provisioning is a Later "agent retry policy" item). This also catches out-of-band `dstack stop`/`delete` of the run by users. -2. No generic server-side endpoint model probe in v1. The backing service owns probes and registration for lifecycle readiness. In the agent path, the agent must make a final model request and include the result in its final report before the server links the final service. - -**Rule: every transition to FAILED issues a one-shot `stop_runs(abort=False)` for a still non-terminal backing run** (RunPipeline completes the termination asynchronously; no waiting needed since FAILED endpoints aren't re-fetched). Stopping a FAILED endpoint later is a no-op unless a linked run still needs cleanup. - -#### `_process_stopping_endpoint` -Two-phase, server-side, and intentionally simple in v1: (1) backing run present and not finished → `stop_runs(abort=False)` once, then no-op wait on subsequent iterations while the normal RunPipeline terminates the run. (2) run finished (or absent/deleted) → write `{status: STOPPED}`, keep the endpoint row visible in history, and emit "Endpoint stopped". Forced abort/escalation for stuck stopping is Later, not required for v1. - -### 6.3 Crash recovery summary - -Replica dies mid-step ⇒ heartbeats stop ⇒ lease expires (≤ ~30s) ⇒ another replica re-fetches the row. V1 handlers are idempotent around the linked `service_run_id`; preset-run-name conflicts are treated as conflicts, not ownership. `EndpointAgentSessionModel` tracks the current Claude process/workspace for an endpoint configuration lifecycle. `EndpointRunSubmissionModel` preserves dstack runs created by that session so recovery does not rely on fragile name heuristics. Agent session budgeting is Later and should be explicit to the agent harness, not a generic endpoint provisioning timeout. Automatic retry is Later. - -### 6.4 Endpoint-level model probes (deferred) - -Do **not** add a generic endpoint probe loop in v1. Service configurations already own probes, and `JobModel.registered` is the service readiness signal after those probes pass. Adding a second endpoint probe loop duplicates service behavior, introduces token/base-URL/proxy/network ambiguity, and makes endpoint lifecycle less conventional for dstack. - -Later, server-side endpoint retry/hardening may add its own explicit verification before re-saving a learned preset or marking an endpoint running again. If/when that is added, prefer reusing the existing in-process probe machinery (`scheduled_tasks/probes.py::_execute_probe` and `get_service_replica_client`) rather than probing through the public service URL. - -### 6.5 Run identity & linkage (decision) - -- **Backing service run name**: the preset path uses `get_endpoint_serving_run_name(endpoint.name)` (`-serving` when it fits, otherwise the endpoint name) because the server submits that service itself. The agent path uses numbered endpoint submissions: `-1`, `-2`, and so on. In both paths, `EndpointModel.service_run_id` is the authoritative link for readiness, URL derivation, logs, and v1 cleanup. Name lookup is not an ownership proof and must not be used to adopt or destroy a run. -- **Endpoint-created run ownership:** `EndpointModel.service_run_id` points to the current/latest service run. `EndpointRunSubmissionModel` records ordered endpoint-submitted run history. Keep `RunModel` endpoint-agnostic. -- Consequences for conservative v1: a non-terminal preset-path service run name not linked by `service_run_id` ⇒ endpoint FAILED with a clear message before preset submission. A terminal conflicting run is handled by the existing run submission path. Agent retry policy and cleanup of non-final agent experiment runs are Later/Path B work. - -### 6.6 Endpoint run submissions - -`EndpointModel.service_run_id` is the current/latest service run and the ownership link for cleanup/readiness/log fallback. `EndpointRunSubmissionModel` records ordered run history submitted by the endpoint: - -```python -class EndpointRunSubmissionModel(BaseModel): - endpoint_id: uuid.UUID - run_id: uuid.UUID - submission_num: int - submitted_at: datetime -``` - -Constraints: -- `UNIQUE(run_id)` so a run is recorded for at most one endpoint. -- `UNIQUE(endpoint_id, submission_num)` so submissions are ordered per endpoint. -- Index `endpoint_id` for endpoint history/cleanup lookup. - -Rules: -- `EndpointModel.service_run_id` remains the single current serving run. -- `EndpointRunSubmissionModel` is endpoint-submitted run history, not the serving-run selector. -- Preset-path run-name lookup remains conflict detection for non-terminal runs only. CLI/user confirmation may stop that conflicting run, but the background worker must not auto-adopt or auto-delete it by name. - -Implementation notes: -- `runs.apply_plan()` can commit internally, so preset-path endpoint code records the submission immediately after a run is accepted and before moving on. -- In the agent path, Claude records every submitted dev environment, task, or service in `submissions.jsonl`. The server reconciles strict numbered submitted run names and reported run IDs into `EndpointRunSubmissionModel` rows; non-strict names are ignored for ownership and cleanup. - ---- - -## 7. Preset subsystem - -Storage and parsing live in `src/dstack/_internal/server/services/endpoints/presets.py`. -Preset matching and run-plan construction live in `src/dstack/_internal/server/services/endpoints/planning.py`. - -### 7.1 What a preset is - -A preset is a **model-level set of tested deployment recipes**. It is not a single generated name and it is not only a service YAML. The model is the lookup key; each recipe has an internal `id`. - -In v1 the local file is a small wrapper around: -- `model`: the endpoint model this preset satisfies and the matching key; -- `recipes`: one or more tested deployment recipes for the model; -- `recipes[*].service`: the serving recipe, service shape, and scheduling resources used for reuse/offer matching; -- `recipes[*].validations`: exact verified hardware evidence from successful deployments, ordered to match `service.replica_groups`. - -The wrapper is deliberate: a recipe is compiled directly into a normal `ServiceConfiguration` before calling the existing run planner. There is no new scheduler path. The service section owns resources because that is already how dstack services describe placement; validations are evidence, not scheduling input. - -Homogeneous service example (`qwen3-32b-vllm-h100x4.dstack.yml`): no explicit `service.replicas` list means there is one implicit service replica group. `validations[0].replicas[0].resources` records the exact resources of each running replica observed when the recipe was verified. - -```yaml -type: endpoint-preset -model: Qwen/Qwen3-32B -recipes: - - id: 8f3a12c4 - service: - image: vllm/vllm-openai:latest - commands: - - | - vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 \ - --tensor-parallel-size $DSTACK_GPUS_NUM - port: 8000 - model: Qwen/Qwen3-32B - env: - - HF_TOKEN - volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true - resources: - shm_size: 16GB - gpu: nvidia:H100:4 - validations: - - replicas: - - resources: - - cpu: 64 - memory: 512GB - disk: 200GB - gpu: nvidia:H100:80GB:4 -``` - -Replica-group example: `validations[*].replicas` is sorted in exactly the same order as `service.replica_groups`. There is no separate group name because service replica-group order is already explicit. This supports existing dstack services with different resource specs per group without making v1 responsible for inventing advanced serving architectures. - -```yaml -type: endpoint-preset -model: Qwen/Qwen3-32B -recipes: - - id: 4c38b901 - service: - port: 8000 - model: Qwen/Qwen3-32B - replicas: - - name: router - count: 1 - image: ghcr.io/example/router:latest - commands: - - python router.py - resources: - cpu: 4 - - name: worker - count: 2 - image: vllm/vllm-openai:latest - commands: - - vllm serve Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 - resources: - shm_size: 16GB - gpu: nvidia:H100:4 - validations: - - replicas: - - resources: - - cpu: 8 - memory: 16GB - disk: 100GB - gpu: 0 - - resources: - - cpu: 64 - memory: 512GB - disk: 200GB - gpu: nvidia:H100:80GB:4 - - cpu: 64 - memory: 512GB - disk: 200GB - gpu: nvidia:H100:80GB:4 -``` - -V1 constraints: -- `service` must not contain `name` or `ProfileParams`. Endpoint `ProfileParams` constrain placement/fleets/pricing on top of the recipe service. -- `service` must contain resources either at the top level or on every explicit replica group. Those resources are the scheduling contract used by reuse planning. -- New learned GPU service resources should specify a vendor. The agent harness should write a broad vendor-aware requirement such as `gpu: nvidia:16GB..24GB:1` after it chooses or verifies NVIDIA hardware. The loader preserves existing vendorless local recipes, but explicit vendor conflicts with validation evidence are invalid. -- `validations[*].replicas` order is part of the contract. If `service.replicas` is a list, `validations[*].replicas[i]` corresponds to `service.replicas[i]`. If there is no replica-group list, each validation has exactly one implicit replica group. -- Each validation replica group contains `resources: list[ResourcesSpec]`, one exact entry per verified running replica/instance in that group. -- Hardware alternatives, benchmarked variants, automatic recipe updates, and metrics on validations are Later. - -### 7.2 Interfaces - -```python -class EndpointPreset(CoreModel): - model: str # matching key - recipes: list[EndpointPresetRecipe] - -class EndpointPresetRecipe(CoreModel): - id: str # hash of normalized service recipe in v1 - service: ServiceConfiguration # compiled service config without name/profile fields - validations: list[EndpointPresetValidation] - -class EndpointPresetValidation(CoreModel): - replicas: list[EndpointPresetValidationReplica] - """Ordered to match `ServiceConfiguration.replica_groups`.""" - -class EndpointPresetValidationReplica(CoreModel): - resources: list[ResourcesSpec] # exact resources for running replicas in this group - -class EndpointPresetService(ABC): # storage abstraction only — local dir first, S3/git later - @abstractmethod - async def list_presets(self, project_name: str) -> list[EndpointPreset]: ... - - @abstractmethod - async def get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: ... - - @abstractmethod - async def delete_preset(self, project_name: str, model: str) -> None: ... - - @abstractmethod - async def save_preset(self, project_name: str, preset: EndpointPreset, comments=None) -> EndpointPreset: ... - -class LocalDirEndpointPresetService(EndpointPresetService): - """Reads *.yml/*.yaml from settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets". - Re-reads per call; file IO via run_async. - Invalid files are logged and skipped, never fatal.""" - -class EndpointPlanJobOffers(CoreModel): - replica_group: str - offers: list[InstanceOfferWithAvailability] # capped by max_offers - total_offers: int - max_price: Optional[float] - -@dataclass(frozen=True) -class EndpointPresetPlan: # planning.py, not presets.py - preset: EndpointPreset - recipe: EndpointPresetRecipe - run_plan: RunPlan -``` - -Module-level `get_endpoint_preset_service()` returning the configured implementation (test-injectable). - -### 7.3 Matching & submission - -This is endpoint planning/orchestration logic, not `EndpointPresetService` storage logic. - -`planning.py::find_matching_preset_plan(session, project, user, endpoint_name, endpoint_conf, preset_service=None) -> Optional[EndpointPresetPlan]`: -1. Candidates: presets whose top-level `model` equals `endpoint_conf.model` case-insensitively, in sorted file-name order. If there are no candidates, return `None` without refreshing the user's SSH key or calling the run planner. -2. For each candidate recipe, build the merged service config (below), wrap in a `RunSpec` (repo fields `None` → virtual repo; `RunSpec.profile` is Optional), and call `runs.get_plan(session, project, user, run_spec, max_offers=1)`. Use the endpoint's effective `creation_policy`; if omitted, keep the normal run default `reuse-or-create` so elastic fleets may provision new instances. If the user explicitly sets `creation_policy: reuse`, matching becomes existing-instances-only. Match ⇔ every `job_plan` has ≥1 offer with `offer.availability.is_available()`. - - Before plan/submission, mirror the run router's behavior: if the creator user has no `ssh_public_key`, call `users.refresh_ssh_key(...)` rather than failing the endpoint. This keeps endpoint apply consistent with `dstack apply` for services. - - **Wrap each candidate in `try/except ServerClientError` → skip**: `get_plan` validates the merged config and can raise for a bad preset; one bad preset must not abort matching. - - Cost note: the planner enumerates cloud backend offers per candidate cloud fleet (`plan.py::get_job_plans` → `find_optimal_fleet_with_offers`) — each candidate evaluation can take seconds. This is acceptable in `/get_plan` and the pipeline only because matching filters by model before planning and uses `max_offers=1`; keep the local preset set small and do not broaden matching into a registry scan in v1. - - Known under-check, accepted for v1 (matching is advisory): `runs.get_plan` produces one representative `JobPlan` per replica group (`replica_num=0`), while validations may record multiple exact running replicas per group. V1 verifies each group has at least one available matching offer for the recipe service resources and leaves full cardinality/capacity checks to the run scheduler. Capacity-aware matching over every validation replica is Later. -3. First match wins. - -`EndpointPresetPlan` contains the selected `EndpointPreset`, selected `EndpointPresetRecipe`, and the `RunPlan` computed from the merged `RunSpec`. The caller must submit that same plan/spec path rather than rebuilding independently, mirroring the run apply split between `runs.get_plan` and `runs.apply_plan`. - -Merged service config (preset → endpoint overrides): -- start from the selected recipe's `ServiceConfiguration`; -- `name = get_endpoint_serving_run_name(endpoint.name)` (the backing service run name); -- merge `endpoint.env` **over** preset env (`Env.update`); endpoint env arrives fully resolved (sentinels rejected at create); -- copy every non-`None` `ProfileParams` field from `EndpointConfiguration` onto the service config (both inherit `ProfileParams`, so this is a field-loop like `RunSpec._merged_profile`, `core/models/runs.py:590-607`); -- do **not** merge or override resources from the endpoint: endpoints do not expose resources in v1, and the recipe service resources are the tested scheduling requirements; -- do not force `creation_policy = reuse`: the normal default `reuse-or-create` is intentional because existing dstack fleets may be elastic and allowed to provision new instances for submitted services/jobs. - -How this runs without the agent: the selected recipe service is already a normal `ServiceConfiguration` without a name/profile. The endpoint layer sets the backing service name and endpoint/profile constraints, then normal run planning/submission decides how many replicas to start and where to place them. `validations` record verified running replica counts/resources at save time; they are evidence and future inspect/JSON input, not a replacement for service `replicas`/autoscaling. - -Submission: `RunSpec(run_name=get_endpoint_serving_run_name(endpoint.name), configuration=merged, ssh_key_pub=None)` as the creator user, using the same policy/SSH-key behavior as the run router (`refresh_ssh_key` if needed; do not let `get_plan` and the final run submission see different specs). If validation still fails, surface that as endpoint FAILED. - -Fleet-drift note: matching is advisory — a fleet can become busy between match and provisioning. No hard endpoint→fleet binding. - -### 7.4 Saving agent-proven presets (v1) - -When an agent-created backing service becomes `RUNNING` through the normal service readiness path, save a sanitized `endpoint-preset` wrapper back into the endpoint project's local preset directory (`/projects//presets`). This makes the second endpoint for the same model in the same project take the deterministic preset path instead of paying the agent again. - -Implementation details: -- Extend `EndpointPresetService` with project-scoped operations such as `save_preset(project_name, preset, comments) -> EndpointPreset`. The local implementation writes a `type: endpoint-preset` YAML file under `settings.SERVER_PROJECTS_DIR_PATH / project_name / "presets"` using an atomic temp-file rename. File names are derived from the model; lookup and merge are by `model`. -- Source of truth is the backing run's stored `RunSpec.configuration` plus the latest successful job submissions after the service has reached `RUNNING`, not the agent's final text. Sanitize before writing: top-level `model` comes from the verified service model; `recipes[*].service` preserves the serving recipe/shape fields (`image`, `commands`, `port`, `model`, `resources`, `volumes`, `replicas`, probes, scaling) but clears `type`, `name`, and `ProfileParams`; `recipes[*].validations[*].replicas[*].resources` are built from jobs grouped by `JobSpec.replica_group` in `ServiceConfiguration.replica_groups` order, deriving each exact instance `ResourcesSpec` from `JobRuntimeData.offer` when present or the backing `InstanceModel.offer` as fallback. Merge env as names only for endpoint-provided keys and redact secret-looking values (`TOKEN`, `KEY`, `SECRET`, `PASSWORD`) to name-only entries. Never write resolved secret values from `EndpointConfiguration.env` to disk. -- Keep YAML metadata minimal in v1. Comments may note the generating endpoint/run, but there is no structured metadata object until a concrete reader uses it. -- Failure to save the preset logs a warning and emits an event, but it does **not** fail an already-running endpoint. Tests should cover successful write, duplicate-name avoidance, invalid destination permissions, and secret redaction. - ---- - -## 8. Agent subsystem - -`src/dstack/_internal/server/services/endpoints/agent/`. - -**Current planning note:** this section has been reset to the CLI-first Path B: the agent uses the real `dstack` CLI in a bounded workspace and performs final functional verification. The server owns endpoint state, run linking, service-readiness bookkeeping, and preset saving; it must not treat its own readiness checks as proof that the requested model works. - -### 8.1 Abstraction - -```python -@dataclass(frozen=True) -class AgentProvisioningResult: - run_id: Optional[uuid.UUID] = None - run_name: Optional[str] = None - error: Optional[str] = None - -class AgentService(ABC): - @abstractmethod - def is_enabled(self) -> bool: ... - @abstractmethod - def get_plan(self) -> AgentPlan: ... - @abstractmethod - async def provision_endpoint( - self, - endpoint_model: EndpointModel, - pipeline_hinter: PipelineHinterProtocol, - ) -> AgentProvisioningResult: ... -``` - -`get_agent_service()` returns the real Claude agent service when enabled, else a disabled stub. Fully injectable for tests (pytest runs with `--disable-socket`; nothing in the test suite may touch the live agent runtime or external APIs). The pipeline requires a successful agent verification report plus the backing `run_id`/`run_name` or an error. Provenance details flow to preset saving/logging only after the server also confirms normal service readiness. - -### 8.2 `ClaudeAgentService` — real agent runtime - -**v1 must use a real Claude agent runtime, not a raw LLM API loop.** The previous raw `anthropic` Messages prototype was removed because it recreated a primitive tool loop instead of giving the agent a real shell/filesystem/code workflow. The current v0 implementation invokes the Claude Code executable as a subprocess from an endpoint workspace. If Claude Code proves unsuitable after live runs, choose another real agent runtime; only fall back to building a custom loop after an explicit design review. - -What this means for v1: -- *Runtime hardening through live runs*: keep the Claude Code subprocess path intentionally simple, then harden it based on real endpoint attempts. It must operate non-interactively, use the real `dstack` CLI, create/edit files, inspect logs/status, preserve artifacts, emit a final machine-readable report, avoid duplicate live agent processes after restarts, and support endpoint stop aborts. -- *CLI-first dstack operations*: planning, submission, status inspection, log reading, and stopping are done through the same `dstack` CLI a user would run: `dstack apply`, `dstack ps`, `dstack run get --json`, `dstack logs`, and `dstack stop`. Do not build duplicate custom tools named `submit_service`, `get_run_status`, `get_run_logs`, etc. -- *Termination contract*: the agent returns a structured final report with success/failure, final service run name/id, final service YAML/content, recipe sources, and verification summary. The agent is responsible for final functional verification. The server does not trust the report blindly; it validates the claimed run identity and project/user ownership before linking the run, then waits for normal dstack service readiness before activating the endpoint. The readiness gate is bookkeeping, not a server-side claim that the model works. -- *Environment hygiene*: the agent runtime receives only the credentials it needs to authenticate to Claude plus the isolated dstack CLI config for the target project. The command/workspace environment remains scrubbed: do not expose server DB credentials, encryption keys, cloud backend credentials, or unrelated server env vars. -- *Abortability and restart*: endpoint stop aborts same-host Claude process groups and stops linked/submitted runs through endpoint submission rows. Same-host restart/resume reuses the existing agent session/workspace. Multi-host restart/supervision, duplicate-process prevention across server instances, and final handoff edge cases still need hardening before production use. - -Packaging requirement: agent runtime dependencies must be installed automatically by normal server deployment paths. The server runtime must not depend on `uv` being available. The Claude Agent SDK currently depends on pydantic v2 while dstack still runs pydantic v1, so v0 invokes the Claude Code executable directly instead of importing the SDK into the server process. Docker release and staging images copy the bundled Claude Code binary into `/usr/local/bin/claude` at build time. `DSTACK_AGENT_CLAUDE_PATH` may point the server at a specific Claude Code executable; when it is unset, `ClaudeAgentService` resolves `claude` from `PATH`. For non-Docker installs, automatic non-Docker packaging for that executable remains a release-blocking follow-up before the agent path can be considered production-ready. - -Packaging gate before enabling `ClaudeAgentService`: -- The server process stays dependency-compatible with pydantic v1 and does not require `uv` at runtime. -- Server Docker release and staging images include the Claude Code executable without an operator shelling into the container to install it manually. -- The availability check used by `get_agent_service()` validates the API key and required `claude` executable, and reports a clear server-install problem if packaging regresses. -- Tests or CI checks cover at least the Python dependency metadata and runtime availability probe; Docker verification is part of the manual e2e runbook if it is too heavy for unit CI. - -### 8.3 CLI-first harness - -The important product work is the harness, not the HTTP loop. The agent should behave like a careful dstack developer using the CLI, with strong context and boundaries: -1. understand the requested model and endpoint constraints; -2. gather grounded recipe + hardware evidence from credible sources; -3. inspect the project context and available fleet/offer envelope through CLI/status output; -4. draft the smallest service YAML that should work; -5. choose the experiment type deliberately: dev environment, task, or service; -6. preview with `dstack apply -f ` before spending GPU time; -7. submit detached with `dstack apply -f -y -d` only when the plan is acceptable within the endpoint/profile envelope; -8. debug with `dstack ps`, `dstack run get --json`, and `dstack logs`; -9. stop ruled-out submitted runs with `dstack stop -y` and let normal run lifecycle handle termination/deletion; -10. finish only after the final service is RUNNING, has a registered replica, exposes a model URL, and the agent has made a final model request proving the requested model is actually served. - -This is intentionally **not** a new server-side mini API for dstack. The harness should provide: -- an endpoint-scoped working directory for service YAMLs, notes, and command transcripts; -- the real `dstack` binary from the running installation; -- a CLI config/context that targets the correct server, project, and endpoint creator user; -- endpoint constraints: model, env names, ProfileParams, preset policy, and allowed fleet/profile envelope; -- recipe grounding guidance and optionally preloaded recipe/context snippets, not a required `find_model_recipes` tool; -- command logging and a `should_abort()` check between command iterations so deletion can interrupt long agent work; -- a structured final-report schema. - -The preset path uses a server-owned deterministic run name from `get_endpoint_serving_run_name(endpoint.name)` because the server submits that service itself. The agent path uses a strict submission naming contract for every dstack run Claude creates: `-1`, `-2`, and so on. Claude still decides what each run is for; framework, hardware, purpose, and role details belong in `submissions.jsonl`, progress messages, `verification.json`, and the final report, not in the run name. The server validates and links the final service by reported run ID, then records it in `EndpointModel.service_run_id` and `EndpointRunSubmissionModel`. - -Dev-environment and task prototyping are not a side path; they are part of the intended harness. The agent should submit a service directly only when the recipe/image/command/resources are already known enough that URL wiring and final service behavior are the main remaining question. If failures show package/runtime compatibility, launch flags, model load, driver/CUDA mismatch, or backend provisioning uncertainty, the next experiment should often use a reusable fleet path, dev environment, or small task when that can avoid repeated cold provisioning. Warm idle instances can speed services too, not only dev environments. This remains agent judgment, not a hardcoded retry rule. For container-only backends that cannot reuse instances, the agent may still use tasks/services, but should avoid resubmitting the same service shape just to learn basic runtime facts. A probe only counts if it exercises the intended serving stack deeply enough to justify promotion; a host-only check such as `nvidia-smi` is not enough. The clean service is still the final proof, and failed final service verification must send the agent back into the evidence loop instead of producing success. - -Endpoint agents do **not** create fleets. This is a product contract, not just prompt preference: the endpoint apply plan and server create path require at least one active existing fleet usable by the endpoint. By default the agent may consider all existing project/imported fleets; endpoint `fleets` narrows that set. If no active fleet exists, the plan/create path reports "The project has no fleets. Create one before submitting an endpoint." If `fleets` is set but none match, it reports that no fleets match the endpoint configuration. - -Within that contract, fleet/backend/hardware choice is still the agent's decision. It should inspect existing fleets, pass applicable `--fleet` filters to `dstack offer`, plan previews, and submitted runs, and choose among the allowed fleets based on the model, endpoint/profile constraints, offer envelope, current fleet state, and experiment. The agent must not create, delete, apply, or edit fleets, including `nodes`, `target`, `idle_duration`, backends, resources, max nodes, or ownership. If the allowed fleets cannot support a useful experiment, it should fail with clear evidence for the user/admin. - -Driver/runtime compatibility is a first-class uncertainty. Some backends expose different host NVIDIA driver/CUDA compatibility across provider pools, regions, GPU generations, or individual nodes, and dstack offers/fleets may not expose that version. The agent may estimate likelihood from provider/GPU/region evidence, but when CUDA/runtime compatibility matters it should gather evidence (`nvidia-smi`, run logs, dev environment/task probe, or known-compatible image/package sources) before trusting unpinned framework installs. This evidence belongs in agent artifacts and future validation metadata; it must not become a fake scheduling constraint unless the backend/user constraints can actually enforce it. - -For v1, advanced P/D disaggregation, multi-service routers/workers, load benchmarking, and autoscaling tuning are Later unless needed to make the requested model serve at all. - -### 8.4 Prompt, recipe grounding & vendored context - -Runtime context layout: - -- `resources/system_prompt.md` — endpoint-specific mission and protocol only: use the real CLI, load `/dstack` and `/dstack-prototyping`, honor endpoint constraints, emit free-form natural-language progress updates, verify the final model API request, and return the structured final report. -- repo-root `skills/dstack/SKILL.md` — CLI/config source of truth. -- repo-root `skills/dstack-prototyping/SKILL.md` — reusable research-to-working-workload skill: source order, model-serving recipe selection, hardware fit, dev-environment/task/service experiment choice, failure classification, final service cleanup, and model API verification. - -`pyproject.toml` force-includes repo-root `skills/**` into wheels/sdists. The workspace setup locates that packaged `skills` directory and copies only `dstack` and `dstack-prototyping` into `.claude/skills` for each endpoint-agent run. The prompt explicitly names `/dstack` and `/dstack-prototyping`; do not rely on an operator's user-level Claude/Codex skills. - -`dstack-prototyping` should answer both "how do I experiment on dstack?" and "what should I try deploying for this model/framework/hardware?" It must stay generic to dstack workload prototyping and must not mention endpoint statuses, preset saving, endpoint DB rows, or Claude-specific implementation details. Endpoint-specific requirements stay in `system_prompt.md`. - -Recipe grounding should be source-oriented, not a static recipe encyclopedia. The agent should prefer current primary sources such as model cards, vLLM/SGLang docs, dstack docs/CLI help, and its own command/log evidence. Advanced posts such as Wafer GLM-on-AMD and LMSYS agent-assisted SGLang development are directional for future harness work, not v1 requirements. - -The prompt interpolates endpoint env keys (names only), ProfileParams constraints, and project context. Recipe grounding is discovered by the agent through allowed network/command facilities and recorded in the final report; do not model it as a required `find_model_recipes` function in v1. - -### 8.5 Execution, cost & limits - -- Runs inside the pipeline worker (§6.2) unless the chosen runtime forces a detached process model. Agent execution must not block the event loop; use async subprocess/process supervision or a small dedicated executor with bounded concurrency, not the shared 128-thread default executor. -- Between agent/runtime steps, or through runtime cancellation hooks, the service checks `should_abort()` (cheap SELECT of stop intent + `lock_token` sanity) and exits early on stop. -- Do **not** add generic endpoint provisioning timeouts, unused turn counters, or a public agent-budget field in v1. Agent budget/cost governance is Later and must be added only with durable per-session accounting and an actually enforced runtime contract. -- Model via `DSTACK_AGENT_ANTHROPIC_MODEL` if the selected runtime supports explicit model selection (default: `claude-opus-4-8`; the default agent should bias toward the strongest Anthropic model because endpoint provisioning is an expensive, tool-heavy deployment investigation). -- Concurrency bound = pipeline `workers_num` per replica (4). -- Observability: log each command + final summary at INFO with the endpoint id; store command transcripts under the endpoint workspace or log service for debugging; store the final agent summary in `status_message` on failure; store `recipe_sources` in the saved preset comments on success. Cost accounting/events → Later. - ---- - -## 9. Settings, packaging, docs - -New in `src/dstack/_internal/server/settings.py` (documented in `mkdocs/docs/reference/env.md`, as the module docstring requires): - -| env var | constant | default | notes | -|---|---|---|---| -| `DSTACK_AGENT_ANTHROPIC_API_KEY` | `AGENT_ANTHROPIC_API_KEY` | `None` | as specified by the requirement (agent-scoped, hence no `_SERVER_`); presence ⇒ agent enabled | -| `DSTACK_AGENT_CLAUDE_PATH` | `AGENT_CLAUDE_PATH` | `None` | optional path to the Claude Code executable; falls back to resolving `claude` from `PATH` | -| `DSTACK_AGENT_ANTHROPIC_MODEL` | `AGENT_ANTHROPIC_MODEL` | `claude-opus-4-8` | override only if the operator intentionally wants a cheaper/faster model | - -Plain module constants in `settings.py`, not env-configurable in v1: `SERVER_PROJECTS_DIR_PATH = /projects`, `ENDPOINT_RUNNING_CHECK_INTERVAL = 60s`. - -Packaging: the selected real agent runtime dependency/binary is installed automatically without contaminating the server Python environment and without requiring `uv` at runtime. For Docker this means copying the bundled Claude Code executable into the image at build time. For non-Docker server installs, automatic packaging of the `claude` executable must be solved before the agent path is production-ready. Treat this as a release blocker for enabling the Claude agent path, not as optional docs. - -Docs: `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._internal.core.models.endpoints.EndpointConfiguration` (processed by `scripts/docs/gen_schema_reference.py`), added to `mkdocs.yml` nav and the `reference/dstack.yml.md` index; env vars in `reference/env.md`; a short concepts page ("Endpoints — experimental") once the feature works end-to-end. - ---- - -## 10. Implementation milestones (each ≈ one PR) - -**M1 — resource skeleton (no processing).** `core/models/endpoints.py` + registration in `configurations.py`; `EndpointModel` + migration + events wiring; `EndpointPlan` models with the `none` provisioning branch; `services/endpoints/__init__.py` (create/stop/list/get/get_plan shell, status switch, events); schemas + router + `app.py`; `_endpoints.py` API client group; CLI configurator + `EndpointCommand`. Endpoints can be planned/created and sit SUBMITTED forever. Tests: router CRUD, name uniqueness/reapply semantics, configurator parse, sentinel rejection, plan output for no preset/no agent. -**M2 — pipeline & lifecycle.** `background/pipeline_tasks/endpoints.py` + registration; the full state machine of §6.2 with `AgentService`/`EndpointPresetService` as interfaces with disabled/empty defaults (so: SUBMITTED→FAILED "nothing configured", preset-path run-name conflict detection, RUNNING backing-run liveness through `service_run_id`, simple two-phase server-side stop of the linked service run, and FAILED teardown rule). Tests: fetcher query (sqlite+postgres, lock claims, RUNNING cadence, STOPPING handling), every worker transition with fakes, conflict detection, linked-run cleanup. Do not test name-based adoption as ownership. -**M3 — presets + plan offers.** `presets.py` (`EndpointPresetService`, `LocalDirEndpointPresetService`, project-scoped storage/parsing only under `/projects//presets`), `endpoint-preset` wrapper parsing (`recipes[*].service` + ordered `validations`), legacy preset conversion, `planning.py` matching via `get_plan` using the endpoint's effective `creation_policy` (default `reuse-or-create`, so elastic fleets are allowed), per-recipe `ServerClientError`/unresolved-env skip, config merge, submission path, `EndpointPlan.provisioning_plan=type:"preset"` only for provisionable recipes with offer summary, save-preset interface + sanitizer/merge. Tests: matching unit tests with testing factories (project/fleet/instance), project isolation, merge semantics, invalid preset files skipped, bad-preset-skip, no-offer preset falls through to agent when policy allows, plan prints selected preset/recipe/offers, secret redaction, atomic write. -**M4 — agent + endpoint logs.** Build on `service_run_id` + `EndpointRunSubmissionModel` so only the latest run serves while submission history is preserved. Continue hardening `ClaudeAgentService` around the v0 Claude Code subprocess runtime: workspace/process handling, scrubbed environment, structured final report artifact, vendored context, `EndpointPlan.provisioning_plan=type:"agent"`, agent settings, automatically installed runtime dependencies, `should_abort`/process cancellation, successful-agent preset save on `RUNNING`, `dstack endpoint logs`, and non-detached endpoint apply following server-side logs/status without attach. Tests should cover real runtime integration boundaries actually used by `ClaudeAgentService`, final-report parsing, runtime availability packaging checks for the `server`/`all` extras, endpoint logs command behavior, apply does not call attach, fake-runtime service integration, and FakeAgentService pipeline integration; mocked network/CLI where possible. Manual e2e: local `uv` server install plus server Docker image both reach the same runtime availability state with only `DSTACK_AGENT_ANTHROPIC_API_KEY` configured. -**M5 — docs & polish.** Reference page, env.md, concepts page, `dstack endpoint` help texts, endpoint plan rendering polish, example presets in docs (not shipped as defaults — §11 Q7), manual e2e runbook (local server + real fleet + one preset + one agent deploy). - ---- - -## 11. Key open questions — with recommended answers - -1. **Run naming & history.** Q: how to link endpoint↔run and what happens on retries? **A:** preset-backed services use `-serving` when valid because the server submits them. Agent-backed dstack runs use numbered endpoint submissions (`-1`, `-2`, ...), while the final serving run is linked by reported `run_id`. `service_run_id` is authoritative for the serving run; name lookup is conflict detection only for preset submission and strict submission reconciliation in the agent path. `EndpointRunSubmissionModel(endpoint_id, run_id, submission_num, submitted_at)` records endpoint-submitted run history without coupling `RunModel` to endpoints. Agent retry policy → Later. -2. **Who owns the backing run?** Q: which UserModel submits it (no system user exists)? **A:** the endpoint's creator (`user_id` on the row) — correct attribution in events/quotas; mirror the run router by refreshing the user's server-managed SSH key before `get_plan`/submission if missing. A first-class service account → Later. -3. **Where does the agent execute?** Q: inside the pipeline worker vs a detached task with its own heartbeat bookkeeping? **A:** v0 runs inside the worker `process()` under the Heartbeater lease, with `workers_num=4` as the concurrency bound. Same-host process abort and same-session restart/resume are implemented, which is enough to keep testing the real loop. Multi-host supervision and final handoff recovery remain hardening items. -4. **Which Claude runtime?** **A: v0 uses Claude Code headless/subprocess after rejecting the raw `anthropic` Messages loop.** Continue validating non-interactive server execution, cancellation, workspace/env isolation, final artifact extraction, restart behavior, and packaging. If Claude Code proves unsuitable after real deployments, choose another real agent runtime before adding more harness complexity. -5. **Permissions.** Q: `ProjectMember` (volumes) or `ProjectAdmin` (gateways)? **A:** `ProjectMember` — endpoints are project workloads that consume the same quota surface as runs, not shared admin infrastructure. -6. **Preset match semantics.** Q: what does "matches the existing fleets" mean concretely? **A:** A model-matched preset is only **provisionable** when `get_plan` on the merged config returns ≥1 available offer for every job plan using the endpoint's effective `creation_policy`. Default `reuse-or-create` may return offers from elastic fleets that can create new instances; explicit `reuse` restricts to currently existing instances. Only provisionable presets are submitted automatically. If presets match the model but no offers are available, `preset_policy: reuse` stops with no-offers/no-fleets, while `reuse-or-create` falls through to the agent path if available. Advisory, not a binding; known to under-check multi-replica capacity (§7.3), accepted for v1. -7. **Do we ship built-in presets?** **A:** No — empty preset dir by default; docs show copy-paste presets, and successful agent deployments write local learned presets. Shipping curated presets with the wheel makes dstack responsible for third-party image/version rot on every release; revisit with a versioned preset registry (Later). -8. **Feature flag?** **A:** No `DSTACK_FF_*` flag. The feature is inert unless a preset dir is populated or the agent key is set; a config-type flag would have to gate client-side parsing too, which FFs here don't do. Document as experimental. -9. **Env var naming.** **A:** keep `DSTACK_AGENT_ANTHROPIC_API_KEY` exactly as specified (breaks the `DSTACK_SERVER_*` convention deliberately: it namespaces a future family of `DSTACK_AGENT_*` settings). -10. **Plan/apply surface.** **A:** v1 ships an advisory `get_plan` plus create/stop/list/get/logs/preset. `dstack apply` always shows the endpoint plan before confirmation, but submit remains create-and-background-process; no endpoint `apply_plan` or in-place update until Later. -11. **Endpoint updates.** **A:** stop/override only in v1. Later in-place endpoint updates must reuse existing service-run `get_plan`/`apply_plan` and rolling deployment (`deployment_num`, desired replica counts, service registration/probes). Endpoint DB updates need their own `configuration_version`/deployment guard so a stale worker in a multi-server Postgres deployment cannot mark a newer endpoint config running. - ---- - -## 12. Later (explicitly deferred) - -Carried over from the requirements ("later we…") plus deferrals made above: - -- **Preset sources beyond local dir**: S3, git repo (reuse the `services/templates.py` clone/TTL machinery), HTTP registry; richer validation metadata (version pins, observed offer, benchmark results), signed/curated preset channels; shipping default presets; capacity-aware matching over every recorded validation replica and exact offer identity. -- **Preset update policy**: allow the agent to update/replace an existing learned preset only as a repair path, not as optimization. The harness must first prove that the old preset is not reproducible under the conditions it claimed to support (for example same model family/recipe constraints and compatible tested resource topology, but repeated normal preset-path attempts fail for reasons attributable to the preset rather than transient capacity/no-offers/user constraints). Only then may it save a replacement or new version, with evidence of the failed reproducibility attempt and the newly verified deployment. V1 learned presets stay append-only/no-overwrite until this policy is designed and tested on real failures. -- **Explicit recipe selection**: allow endpoint configs to specify `recipe: ` later when a model has multiple learned recipes. V1 does not expose this field; it tries stored recipes in order and uses the first recipe with available offers under the endpoint's effective fleets/profile constraints. -- **Automatic provisioning retry policy**: retry failed provisioning through SUBMITTED, backoff, retry history retention, and distinct submitted run names if preserving failed submissions becomes important. -- **Agent retry policy for RUNNING endpoints**: instead of FAILED on health-check failure, re-invoke the agent to diagnose/redeploy; backoff policy; `retry` ProfileParams semantics for endpoints. -- **Alternative agent runtime** — revisit if the first real runtime cannot meet server requirements for non-interactive execution, packaging, cancellation, workspace/env isolation, artifact capture, and cost accounting. -- **Other `AgentService` implementations** (OpenAI-compatible, Bedrock/Vertex via the SDK's provider clients, self-hosted). -- **Richer service creation**: deepen `dstack-prototyping` through real endpoint-agent traces; richer attach/SSH automation for dev environments; benchmarking before marking an endpoint running; autoscaling/replicas/gateway/domain decisions; quantization variant selection; multi-node deployments; P/D disaggregation and other multi-component serving topologies; advanced SGLang development/deployment harnesses inspired by the 2026-07-02 LMSYS agent-assisted SGLang workflow. -- **Workload-aware optimization**: benchmark and tune endpoints by workload profile (chatbot, RAG, code generation, long-form generation), measuring TTFT, ITL, throughput, end-to-end latency, quality, and cost/token. Product references such as Modal Auto Endpoints, Makora, and Runpod Overdrive are useful directionally, but v1 must stop at verified functional deployment and reproducible preset recipes. -- **In-place update / plan-apply**: server-side endpoint `get_plan`/`apply_plan`, model or config changes via the existing service-run `apply_plan` + rolling deployment machinery, no stop/recreate for simple changes; endpoint row updates protected by a `configuration_version`/deployment guard for multi-server safety. -- **Endpoint-level resources**: optional user-specified resource requirements for endpoint configs. Deferred because replica groups make merge semantics non-trivial. If added, the field must be enforced by preset planning and agent-submitted YAML, must define how it combines with preset/service replica-group resources, and must be reflected in the apply plan without pretending it is final tested hardware. -- **Deletion escalation**: forced abort / operator-facing stuck-deletion handling if a backing run remains terminating beyond the normal RunPipeline retry window. -- **Plugins**: add an `EndpointSpec` to the `ApplySpec` TypeVar (`dstack/plugins/_models.py:8`) so plugin policies can reason about endpoint-level intent directly. Backing service runs still use normal run policies in v1. -- **Frontend UI**: endpoints page in the server frontend. No frontend work is required in v1. -- **Richer progress surfaces**: agent transcript/event streaming beyond concise status/log-following; structured display of recipe evidence and harness decisions in CLI/UI. -- **Cost & governance**: per-project agent budgets, token/cost accounting per endpoint, API key via the encrypted secrets service (`services/secrets.py`) instead of a server-wide env var, audit events for every agent tool call. -- **Agent hardening**: policy hooks on tool calls, sandboxed execution, structured-output enforcement of the final report beyond the `finish` tool. -- **Vendored recipe snapshots** for air-gapped servers (Apache-2.0 attribution), refresh tooling. -- **Model catalog UX**: `dstack endpoint models` listing known-deployable models from presets + recipes indexes. - ---- - -## 13. Risks & mitigations (summary) - -| risk | mitigation in this plan | -|---|---| -| Replica death mid-agent-run duplicates deployments / spend | `service_run_id` points to the latest run and `EndpointRunSubmissionModel` records accepted endpoint-submitted runs. The heartbeater lease prevents duplicate live workers while the replica lives; after a crash, the next worker reconciles by linked run/submission history. Name lookup is conflict-only (§6.2, 6.3, 6.5) | -| Paid crash loop (pipeline re-fetch + flaky agent) | no automatic retry in v1 + terminal FAILED stops re-fetching; real agent loop must add explicit token/cost/wall-clock budgets before it can spend on experiments | -| Adopting/destroying an unrelated user run with the same name | name lookup is conflict detection only; ownership must come from `service_run_id` or `EndpointRunSubmissionModel` rows, never from name/user/timestamp heuristics (§6.5) | -| One-shot stop+delete of runs always raising | endpoint stop remains two-phase stop → wait-until-finished → stopped. The CLI-agent should use normal `dstack stop -y` for ruled-out submitted runs and let run lifecycle handle termination (§2.5, §8.3) | -| GPU-run leaks on terminal FAILED | every FAILED transition issues `stop_runs` for a still non-terminal backing run (§6.2) | -| Transaction/lock corruption submitting backing runs from the worker | fresh `get_session_ctx()` for all runs-service calls; endpoint row writes stay token-guarded; any interim progress update leaves lock columns untouched (§6.1) | -| Prompt injection via fetched recipes → server compromise | command execution runs in an endpoint-scoped workspace with a scrubbed environment; the agent uses normal `dstack` CLI rather than server internals; server secrets/DB/cloud credentials and the agent API key are not exposed to commands (§8.2, 8.3) | -| Agent-learned preset writes resolved secrets to disk | preset sanitizer clears `name`, writes endpoint env as names only, redacts secret-looking values, and tests secret redaction (§7.4) | -| Event-loop starvation from long agent IO | supervise the agent runtime with async subprocess/process APIs or a bounded dedicated executor; never the shared default executor; small `workers_num` | -| `RunStatus.RUNNING` ≠ model ready | endpoint `RUNNING` requires a registered running service replica and `ServiceSpec.model.base_url`; endpoint v1 does not add a parallel health probe (§6.4) | -| `MissingGreenlet` on lazy `project.backends` in the worker | refetch joinedloads chain `project → backends` (§6.1) | -| `register_service` raises synchronously during backing run submission (gateway config issues) | caught as submission failure ⇒ FAILED with message; agent prompt includes gateway context | -| Agent deploys a service without `model:` → no model URL to surface | prompt requires `model:`; agent final verification must use the model endpoint, and server readiness validation refuses to mark the endpoint running unless the backing service exposes `ServiceSpec.model.base_url` (§6.2, §8.3) | -| Task-first degenerates into a batch script | prompt/skill require long-lived interactive probes when attach/SSH is available, and the endpoint agent's local `dstack` wrapper now rejects batch-style probe tasks before submission. Unit coverage verifies rejection of a task that packs host/framework/server/curl checks into `commands` and acceptance of a `sleep infinity` probe task. Live e2e must still prove the agent reacts correctly by attaching/SSHing into the task and promoting a clean service | -| Repeated model downloads because cache mounts are omitted | prompt/skill now require optional instance cache mounts for Hugging Face-style model-serving probes/services when useful, or an explicit artifact explaining why cache mounts do not help for that backend/model. Presets preserve useful `volumes` from the verified service | -| Preset/fleet drift between match and provisioning | matching is advisory; run scheduling is source of truth | -| Agent runtime dependency missing in deployment | selected runtime dependencies are included in normal server install paths and Docker images; release Docker uses `dstack[all]`, staging uses `uv sync --extra all`, and local server installs must not require manual runtime installation (§8.2, 9) | -| Missing migration passes unit tests silently | migration is an explicit M1 review item; postgres-parametrized tests | diff --git a/endpoint-implementation-research/background-loop.md b/endpoint-implementation-research/background-loop.md deleted file mode 100644 index 11e3a8daea..0000000000 --- a/endpoint-implementation-research/background-loop.md +++ /dev/null @@ -1,241 +0,0 @@ -# background-loop - -## Summary -dstack's server background processing has two families, both started from the FastAPI lifespan in src/dstack/_internal/server/app.py (lines 178-184, gated by settings.SERVER_BACKGROUND_PROCESSING_ENABLED): (1) "pipeline tasks" — per-DB-model fetch/worker/heartbeat pipelines (background/pipeline_tasks/base.py) that claim rows with durable DB lock columns (lock_expires_at/lock_token/lock_owner) plus SELECT ... FOR UPDATE SKIP LOCKED, and (2) "scheduled tasks" — APScheduler IntervalTrigger jobs for infrequent, idempotent work. Multi-replica safety comes from the lock columns (not in-memory locks): a replica heartbeats lock_expires_at forward every ~1s while processing, and every mutation is guarded by `WHERE id = :id AND lock_token = :token`; if a replica dies, the lock expires and another replica's fetcher (which selects rows with `lock_expires_at IS NULL OR lock_expires_at < now`, ordered by last_processed_at ASC) takes over with a new token. Long-running work (instance provisioning, 10-55 min) is handled by (a) indefinite heartbeat extension during one `process()` call and (b) chunking across pipeline iterations via a status state machine (PENDING -> PROVISIONING -> per-iteration readiness checks against a deadline). A new `process_endpoints` task should be a new `EndpointPipeline` copied from the VolumePipeline skeleton (the simplest single-model example), registered in PipelineManager, with an EndpointModel using PipelineModelMixin + last_processed_at + a partial fetch index and an alembic migration. - -## Key files -- src/dstack/_internal/server/background/pipeline_tasks/base.py — Pipeline, Fetcher, Worker, Heartbeater, PipelineItem, PipelineModel, ItemUpdateMap, NOW_PLACEHOLDER, set_unlock_update_map_fields, set_processed_update_map_fields, resolve_now_placeholders, log_lock_token_mismatch, log_lock_token_changed_after_processing, log_lock_token_changed_on_reset — The generic pipeline framework every processing task follows; 483 lines, fully read. -- src/dstack/_internal/server/background/pipeline_tasks/__init__.py — PipelineManager, PipelineHinter, get_pipeline_manager, start_pipeline_tasks, PipelineManager.register_pipeline — Where a new EndpointPipeline must be registered (builtin list at lines 35-48). -- src/dstack/_internal/server/background/pipeline_tasks/volumes.py — VolumePipeline, VolumeFetcher.fetch, VolumeWorker.process, _refetch_locked_volume, _apply_process_result, _VolumeUpdateMap, _ProcessResult — Cleanest single-model pipeline; the recommended template for process_endpoints (SUBMITTED->ACTIVE/FAILED via backend calls). -- src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py — RunPipeline, RunFetcher.fetch, RunWorker.process, _lock_related_jobs, _unlock_related_jobs, _reset_run_lock_for_retry — Example of cross-model child-row locking (run locks its jobs with the same lock_token) and per-status dispatch. -- src/dstack/_internal/server/background/pipeline_tasks/fleets.py — FleetPipeline, FleetFetcher.fetch, _lock_fleet_instances_for_processing, _apply_process_result — Second cross-model locking example; also shows exponential consolidation retry delays pattern (_CONSOLIDATION_RETRY_DELAYS). -- src/dstack/_internal/server/services/locking.py — get_locker(dialect_name), ResourceLocker, InMemoryResourceLocker, DummyResourceLocker, advisory_lock_ctx, try_advisory_lock_ctx, string_to_lock_id — In-memory lockset for SQLite, no-op dummy for Postgres; Postgres advisory locks for one-off critical sections. NOT the multi-replica mechanism for row processing. -- src/dstack/_internal/server/models.py — PipelineModelMixin (lines 204-207), RunModel (405), VolumeModel (951), ix_runs_pipeline_fetch_q index (464-472) — lock_expires_at/lock_token/lock_owner mixin + per-model last_processed_at + partial fetch index; EndpointModel must replicate this. -- src/dstack/_internal/server/background/scheduled_tasks/__init__.py — start_scheduled_tasks, get_scheduler, AsyncIOScheduler — APScheduler interval/date jobs; per-replica, no cross-replica dedup; for infrequent idempotent work only. -- src/dstack/_internal/server/services/pipelines.py — PipelineHinterProtocol, get_pipeline_hinter — FastAPI dependency for API handlers to hint fetchers after submitting a row (reduces processing latency). -- src/dstack/_internal/server/app.py — lifespan (start_scheduled_tasks/start_pipeline_tasks at 178-184, shutdown/drain at 206-214, default ThreadPoolExecutor at 123-124) — Startup/shutdown wiring; app.state.pipeline_manager set at line 181. -- src/dstack/_internal/server/db.py — get_db, get_session_ctx, get_session, Database.dialect_name, is_db_sqlite, is_db_postgres, sqlite_commit — Session helpers used by all fetchers/workers; get_session_ctx commits on clean exit; autoflush disabled. -- src/dstack/_internal/server/settings.py — SERVER_BACKGROUND_PROCESSING_ENABLED/DISABLED (47-50), SERVER_EXECUTOR_MAX_WORKERS (52), MAX_OFFERS_TRIED (54) — Only env vars controlling background processing; no per-pipeline tuning env vars exist. -- src/dstack/_internal/server/migrations/versions/2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py — upgrade/downgrade — Most recent 'add a pipeline' migration: adds lock columns + status + last_processed_at, backfills last_processed_at=created_at. Template for endpoints migration. -- src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py — offers loop capped by settings.MAX_OFFERS_TRIED (line 121), compute.create_instance via run_async (172) — Long-running (minutes) process() example, kept alive by heartbeater. -- src/dstack/_internal/server/background/scheduled_tasks/idle_volumes.py — process_idle_volumes — Scheduled-task skeleton: lockset + FOR UPDATE SKIP LOCKED + respects pipeline rows via lock_expires_at IS NULL. -- src/tests/_internal/server/background/pipeline_tasks/test_volumes.py — TestVolumeFetcher, fetcher/worker fixtures, _volume_to_pipeline_item — Test conventions: instantiate Fetcher/Worker with Mock queue/heartbeater, parametrize test_db over ['sqlite','postgres']. - -## Details -All paths relative to /Users/dstack/dstack. Everything below was read from the working tree (branch master, commit 28ea5f86f era). - -## 1. Two background-task families and how they start - -`src/dstack/_internal/server/app.py` lifespan: -- line 123-124: `server_executor = ThreadPoolExecutor(max_workers=settings.SERVER_EXECUTOR_MAX_WORKERS)`; `asyncio.get_running_loop().set_default_executor(server_executor)` — this is the pool `run_async` uses. -- lines 176-184: -```python -if settings.SERVER_BACKGROUND_PROCESSING_ENABLED: - scheduler = start_scheduled_tasks() - pipeline_manager = start_pipeline_tasks() - app.state.pipeline_manager = pipeline_manager -... -PROBES_SCHEDULER.start() # separate AsyncIOScheduler for probes, started unconditionally -``` -- shutdown (206-214): `pipeline_manager.shutdown()` → `scheduler.shutdown()` → `await pipeline_manager.drain()`. - -### pipeline_tasks (`background/pipeline_tasks/__init__.py`) -`start_pipeline_tasks() -> PipelineManager` (line 106) docstring: "Start tasks processed by fetch-workers pipelines based on db + in-memory queues. Suitable for tasks that run frequently and need to lock rows for a long time." - -`PipelineManager.__init__` (lines 31-49) instantiates and registers 12 builtin pipelines, each constructed as `XxxPipeline(pipeline_hinter=self._hinter)`: ComputeGroupPipeline, FleetPipeline, GatewayPipeline, GatewayReplicaPipeline, JobSubmittedPipeline, JobRunningPipeline, JobTerminatingPipeline, InstancePipeline, PlacementGroupPipeline, RunPipeline, ServiceRouterWorkerSyncPipeline, VolumePipeline. Public `register_pipeline(pipeline)` (line 51) appends and registers with the hinter — this is where `EndpointPipeline` gets added. - -`PipelineHinter` (lines 80-93): `_hint_fetch_map: dict[str, list[Pipeline]]` keyed by `pipeline.hint_fetch_model_name` (a `Model.__name__` string, e.g. `"VolumeModel"`); `hint_fetch(model_name)` calls `pipeline.hint_fetch()` on all pipelines registered for that model (all three job pipelines share `JobModel.__name__`). Singleton via `get_pipeline_manager()` (line 99). - -API handlers obtain the hinter via `get_pipeline_hinter(request: Request) -> PipelineHinterProtocol` in `src/dstack/_internal/server/services/pipelines.py:22` (reads `request.app.state.pipeline_manager`, returns a no-op hinter if background processing is disabled). Real usages: `services/volumes.py:317 pipeline_hinter.hint_fetch(VolumeModel.__name__)` after volume create; `services/runs/__init__.py:825-826, 906`; `services/fleets.py:880, 1112-1113`; `services/gateways/__init__.py:291`. Hints are local-replica only (they just set an asyncio.Event on the fetcher). - -### scheduled_tasks (`background/scheduled_tasks/__init__.py`) -`start_scheduled_tasks() -> AsyncIOScheduler` (line 37) docstring: "Start periodic tasks triggered by apscheduler at specific times/intervals. Suitable for tasks that run infrequently and don't need to lock rows for a long time." Jobs (lines 43-60): `init_gateways_in_background` (DateTrigger, once), `preload_offers_catalog` (DateTrigger + IntervalTrigger(minutes=10)), `process_probes` (seconds=3, jitter=1), `collect_metrics` (10s), `delete_metrics` (5m), `delete_events` (7m), `process_gateways_connections` (15s), `process_idle_volumes` (60s, jitter=10), `delete_instance_healthchecks` (5m), plus prometheus pair if `settings.ENABLE_PROMETHEUS_METRICS`. `max_instances=1` on most — note this is per-process only, NOT cross-replica; every replica runs every scheduled task, so they must be idempotent (e.g. `delete_events` is a bare `DELETE WHERE recorded_at < cutoff`, scheduled_tasks/events.py:13-17). - -## 2. The pipeline framework (base.py) — exact skeleton - -`background/pipeline_tasks/base.py`: - -```python -@dataclass -class PipelineItem: # base.py:34 - __tablename__: str - id: uuid.UUID - lock_expires_at: datetime - lock_token: uuid.UUID - prev_lock_expired: bool # set by fetchers, currently consumed nowhere (verified by grep) - -class PipelineModel(Protocol): # base.py:50 — model must have: - __tablename__: str; __mapper__; __table__ - id: Mapped[uuid.UUID] - lock_expires_at: Mapped[Optional[datetime]] - lock_token: Mapped[Optional[uuid.UUID]] - -class Pipeline(Generic[ItemT], ABC): # base.py:67 - def __init__(self, workers_num, queue_lower_limit_factor, queue_upper_limit_factor, - min_processing_interval: timedelta, lock_timeout: timedelta, - heartbeat_trigger: timedelta) -> None: ... - def start(self): ... # creates asyncio tasks: heartbeater.start(), each worker.start(), fetcher.start() - def shutdown(self): ... # stop flags + cancel tasks - async def drain(self): ... - def hint_fetch(self): self._fetcher.hint() - # abstract: hint_fetch_model_name (property str), _heartbeater, _fetcher, _workers -``` -Queue sizing: `_queue_desired_minsize = ceil(workers_num * queue_lower_limit_factor)`, `_queue_maxsize = ceil(workers_num * queue_upper_limit_factor)`; `asyncio.Queue[ItemT](maxsize=_queue_maxsize)`. - -`Fetcher` (base.py:257): loop — if `qsize >= desired_minsize` sleep `queue_check_delay=1.0`; else `items = await self.fetch(limit=maxsize - qsize)`; on empty fetch, wait on `self._fetch_event` with timeout from `_DEFAULT_FETCH_DELAYS = [0.5, 1, 2, 5]` seconds indexed by consecutive-empty count, ±20% jitter, with a 10% random chance of resetting to the minimal delay (base.py:326-337); `hint()` sets the event. Non-empty: `queue.put_nowait(item)` + `heartbeater.track(item)`. Abstract: `async def fetch(self, limit: int) -> list[ItemT]`. - -`Worker` (base.py:340): `__init__(queue, heartbeater, pipeline_hinter: PipelineHinterProtocol)`; loop: `item = await queue.get()`; `await self.process(item)` inside try/except-log; `finally: await self._heartbeater.untrack(item)`. Abstract: `async def process(self, item: ItemT)`. - -`Heartbeater` (base.py:166): `__init__(model_type: type[PipelineModel], lock_timeout, heartbeat_trigger, heartbeat_delay: float = 1.0)`. Every ~1s: for each tracked item, if `lock_expires_at < now` → untrack + warn ("Failed to heartbeat ... in time"); if `lock_expires_at < now + heartbeat_trigger` → include in one bulk `UPDATE model SET lock_expires_at = now + lock_timeout WHERE (id==i.id AND lock_token==i.lock_token) OR ... RETURNING id` (base.py:227-240); items whose token changed are untracked. So a lock is extended indefinitely while the replica is alive and the worker is still processing — this is what makes minutes-long `process()` calls safe. - -Update-map helpers (base.py:379-483): `NOW_PLACEHOLDER` + `resolve_now_placeholders(update_values, now)` so all timestamps in one apply-transaction share the same `now`; `ItemUpdateMap` TypedDict with `lock_expires_at/lock_token/lock_owner/last_processed_at`; `set_unlock_update_map_fields(m)` sets the three lock fields to None; `set_processed_update_map_fields(m, now=NOW_PLACEHOLDER)` sets `last_processed_at`; standard warn-loggers `log_lock_token_mismatch(logger, item, action="process")`, `log_lock_token_changed_after_processing(logger, item, ...)`, `log_lock_token_changed_on_reset(logger)`. - -## 3. Model-side requirements - -`src/dstack/_internal/server/models.py`: -```python -class PipelineModelMixin: # models.py:204 - lock_expires_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) - lock_token: Mapped[Optional[uuid.UUID]] = mapped_column(UUIDType(binary=False)) - lock_owner: Mapped[Optional[str]] = mapped_column(String(100)) -``` -Models using it: RunModel(405), ServiceRouterWorkerSyncModel(475), JobModel(506), GatewayModel(600), GatewayComputeModel(656), FleetModel(754), InstanceModel(805), VolumeModel(951), PlacementGroupModel(1011), ComputeGroupModel(1047). - -Each pipeline-processed model also declares its own `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime)` (non-null) and a partial index for the fetch query, e.g. RunModel (models.py:464-472): -```python -Index("ix_runs_pipeline_fetch_q", last_processed_at.asc(), - postgresql_where=status.not_in(RunStatus.finished_statuses()), - sqlite_where=status.not_in(RunStatus.finished_statuses())) -``` -New rows are created with `last_processed_at` = submission time (`services/runs/__init__.py:744 last_processed_at=submitted_at`; `services/volumes.py:307 last_processed_at=now`); fetchers treat `last_processed_at == created_at` (or `== submitted_at` for runs) as "never processed → skip the min-interval gate". Some models additionally have `skip_min_processing_interval: Mapped[bool]` (RunModel:432, JobModel, InstanceModel) which fetchers OR into the interval condition and reset to False on fetch. - -Migration template: `migrations/versions/2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py` — batch_alter_table adds `last_processed_at` (NaiveDateTime), `status`, `status_message`, `lock_expires_at`, `lock_token` (UUIDType(binary=False)), `lock_owner` (String(100)); backfills `last_processed_at = created_at`; then makes columns non-null. Migrations live under `src/dstack/_internal/server/migrations/versions//`. - -## 4. Standard fetch skeleton (verified in VolumeFetcher.fetch, volumes.py:135-196; same shape in runs/fleets/instances) - -```python -volume_lock, _ = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__) -async with volume_lock: # asyncio.Lock on SQLite; DummyAsyncLock (no-op) on Postgres - async with get_session_ctx() as session: - now = get_current_datetime() - res = await session.execute( - select(VolumeModel) - .where( - or_(VolumeModel.status == VolumeStatus.SUBMITTED, VolumeModel.to_be_deleted == True), - VolumeModel.deleted == False, - or_(VolumeModel.last_processed_at <= now - self._min_processing_interval, - VolumeModel.last_processed_at == VolumeModel.created_at), - or_(VolumeModel.lock_expires_at.is_(None), VolumeModel.lock_expires_at < now), - or_(VolumeModel.lock_owner.is_(None), VolumeModel.lock_owner == VolumePipeline.__name__), - ) - .order_by(VolumeModel.last_processed_at.asc()) - .limit(limit) - .with_for_update(skip_locked=True, key_share=True, of=VolumeModel) - .options(load_only(VolumeModel.id, VolumeModel.lock_token, VolumeModel.lock_expires_at, ...))) - volume_models = list(res.scalars().all()) - lock_expires_at = get_current_datetime() + self._lock_timeout - lock_token = uuid.uuid4() # ONE token per fetched batch - for m in volume_models: - prev_lock_expired = m.lock_expires_at is not None - m.lock_expires_at = lock_expires_at; m.lock_token = lock_token - m.lock_owner = VolumePipeline.__name__ - items.append(VolumePipelineItem(__tablename__=VolumeModel.__tablename__, id=m.id, ...)) - await session.commit() -return items -``` -Fetchers are decorated `@sentry_utils.instrument_pipeline_task("VolumeFetcher.fetch")` (workers likewise with `"VolumeWorker.process"`); `instrument_pipeline_task(name)` / `instrument_scheduled_task(f)` live in `server/utils/sentry_utils.py:14-19`. - -## 5. Standard worker skeleton (VolumeWorker.process, volumes.py:212-291) - -1. Refetch with full relationships: `select(VolumeModel).where(VolumeModel.id == item.id, VolumeModel.lock_token == item.lock_token).options(joinedload(...))` → `scalar_one_or_none()`; if None → `log_lock_token_mismatch(logger, item)`; return. -2. Do the actual processing as pure-ish functions returning a `_ProcessResult` containing a TypedDict update map (`_VolumeUpdateMap(ItemUpdateMap)` adds `status`, `status_message`, `volume_provisioning_data`, `deleted`, `deleted_at`). Blocking backend/SDK calls go through `run_async` (`src/dstack/_internal/utils/common.py:49-51`: `await asyncio.get_running_loop().run_in_executor(None, partial(func, *args, **kwargs))` — the default executor is the 128-thread pool from app.py:123). -3. Apply: `set_processed_update_map_fields(update_map)`; `set_unlock_update_map_fields(update_map)`; `resolve_now_placeholders(update_map, now=get_current_datetime())`; then -```python -res = await session.execute( - update(VolumeModel) - .where(VolumeModel.id == volume_model.id, VolumeModel.lock_token == volume_model.lock_token) - .values(**update_map).returning(VolumeModel.id)) -if len(list(res.scalars().all())) == 0: - log_lock_token_changed_after_processing(logger, item); return -``` -plus event emission (`services.events.emit(session, msg, actor=events.SystemActor(), targets=[events.Target.from_model(model)])` and per-domain `emit_*_status_change_event`). - -## 6. Cross-model (child row) locking — runs and fleets - -`RunPipeline._lock_related_jobs` (runs/__init__.py:763-810): under the jobs lockset lock, `select(JobModel).where(JobModel.run_id == item.id, status not in JOB_STATUSES_EXCLUDED_FOR_LOCKING, lock free or expired, lock_owner NULL or == RunPipeline.__name__).order_by(JobModel.id).with_for_update(skip_locked=True, key_share=True, of=JobModel)`; then re-selects ALL current eligible job ids — if the locked set != full set, it gives up: `_reset_run_lock_for_retry` (runs/__init__.py:813-835) which keeps `lock_owner` (so the row stays owned by the pipeline and other subsystems stay away), but sets `lock_expires_at=None, lock_token=None, last_processed_at=now` so the item is retried on a later fetch and the heartbeater can no longer touch it. Children get the parent's `lock_expires_at/lock_token` and `lock_owner=RunPipeline.__name__`. On every apply/noop path, `_unlock_related_jobs` (950-969) nulls the three lock columns `WHERE id IN locked AND lock_token == item.lock_token AND lock_owner == RunPipeline.__name__`. FleetPipeline does the same for InstanceModel (`fleets.py:374-445`, unlock at 762-779), and InstanceFetcher avoids fighting the fleet by requiring `FleetModel.lock_owner IS NULL` for instances in a fleet (instances/__init__.py:212-224). - -Related: run/fleet workers unlock/update parent + children in ONE transaction; job update rows are built with `set_unlock_update_map_fields`/`set_processed_update_map_fields` per row and executed as bulk `await session.execute(update(JobModel), job_update_rows)` (runs/__init__.py:588-589). - -## 7. Locking service — exact API (`src/dstack/_internal/server/services/locking.py`) - -- `get_locker(dialect_name: str) -> ResourceLocker` (line 175): returns module-level `InMemoryResourceLocker` for `"sqlite"`, else `DummyResourceLocker` ("We could use an in-memory locker on Postgres but it can lead to unnecessary lock contention, so we use a dummy locker that does not take any locks."). NOTE: it takes `dialect_name` as a required arg — call sites are all `get_locker(get_db().dialect_name)`. -- `ResourceLocker.get_lockset(namespace: str) -> tuple[LocksetLock, Lockset]` — a guard lock plus a set of locked keys; `lock_ctx(namespace, keys)` acquires all keys (keys must be sorted to avoid deadlock; implemented by `_wait_to_lock_many(lock, locked, keys, delay=0.1)`). -- `string_to_lock_id(s) -> int` (sha256 mod 2**63); `advisory_lock_ctx(bind, dialect_name, resource)` (line 125) — `pg_advisory_lock`/`pg_advisory_unlock`, NO-OP on SQLite, with documented footguns (must release on the same connection; don't commit inside when bind is an AsyncSession); `try_advisory_lock_ctx` (line 156) yields a bool. Used for `migrate()` (db.py:83) and `server_init` (app.py:140). - -**How multi-replica double-processing is actually prevented for pipelines**: not by the locker. It's (a) the durable `lock_expires_at`/`lock_token`/`lock_owner` columns claimed in the fetch transaction, (b) `WITH FOR UPDATE SKIP LOCKED` making concurrent Postgres fetch transactions skip each other's rows mid-claim, and (c) every subsequent write being conditioned on `lock_token`. On SQLite (single replica by definition) `with_for_update` is a no-op and the in-memory lockset lock serializes fetchers/scheduled-task claimers within the process instead. - -## 8. Replica death / stale lock recovery - -If a replica dies mid-processing: heartbeats stop → `lock_expires_at` (20-40s in the future) passes → any replica's fetcher matches the row again via `or_(lock_expires_at.is_(None), lock_expires_at < now)` and issues a NEW `lock_token`; `prev_lock_expired=True` is recorded on the item (currently informational only). If the dead replica's worker somehow finishes later, its `UPDATE ... WHERE lock_token == old_token` matches 0 rows and is logged, not applied. There is no separate reaper/janitor; recovery latency ≈ remaining lock_timeout. `ORDER BY last_processed_at ASC` guarantees stalest-first pickup. If `process()` raises, the Worker logs and unt racks; the row simply stays locked until expiry (retry latency ≈ lock_timeout). - -## 9. Long-running operations (minutes+) - -Two mechanisms, both exemplified by instances: -1. **Heartbeat-extended single call**: `InstancePipeline` PENDING processing (instances/cloud_provisioning.py) loops over offers, each `compute.create_instance` executed via `run_async` (line 172), loop capped by `settings.MAX_OFFERS_TRIED` (line 121, default 25, "Limit number of offers tried to prevent long-running processing in case all offers fail"). The Heartbeater keeps extending the row lock every ~1s check for as long as needed. `JobSubmittedPipeline` similarly calls `run_async(compute...)` at jobs_submitted.py:1816, 2287, 2309. -2. **Chunked state machine across iterations**: after `create_instance` returns, the worker only writes `status=PROVISIONING` + `job_provisioning_data` + `started_at=NOW_PLACEHOLDER` and unlocks (cloud_provisioning.py:208-231). Each subsequent pipeline iteration (~min_processing_interval) re-picks the PROVISIONING row and checks readiness against `get_provisioning_deadline` (instances/check.py:200-204, 297-301), with per-backend timeouts from `get_provisioning_timeout(backend_type, instance_type_name)` in `background/pipeline_tasks/common.py:6` (10 min default, up to 55 min for Vultr bare metal). **This is the pattern to copy for an endpoint LLM-agent flow: persist per-phase status (e.g. SUBMITTED → PROVISIONING/DEPLOYING → health-checking) so each `process()` is resumable if a replica dies, rather than one hours-long process() call.** - -## 10. Pipeline tuning defaults (constructor kwargs; no env vars) - -| Pipeline | workers_num | min_processing_interval | lock_timeout | heartbeat_trigger | -|---|---|---|---|---| -| RunPipeline (runs/__init__.py:56) | 10 | 5s (x2 for non SUBMITTED/TERMINATING) | 30s | 15s | -| JobSubmittedPipeline (jobs_submitted.py:158) | 40 | 4s | 40s | 20s | -| JobRunningPipeline (jobs_running.py:138) | 20 | 5s | 30s | 15s | -| JobTerminatingPipeline (jobs_terminating.py:91) | 20 | 2s | 30s | 15s | -| InstancePipeline (instances/__init__.py:81) | 20 | 7s (x2 for idle/busy) | 30s | 15s | -| FleetPipeline (fleets.py:69) | 10 | 15s | 20s | 10s | -| VolumePipeline (volumes.py:61) | 10 | 15s | 30s | 15s | -| GatewayPipeline (gateways.py:62) | 10 | 15s | 30s | 15s | -| GatewayReplicaPipeline (gateway_replicas.py:56) | 10 | 15s | 30s | 15s | -| ComputeGroupPipeline (compute_groups.py:49) | 10 | 15s | 30s | 15s | -| PlacementGroupPipeline (placement_groups.py:46) | 10 | 15s | 30s | 15s | -| ServiceRouterWorkerSyncPipeline (service_router_worker_sync.py:54) | 8 | 5s | 25s | 10s | -All use queue_lower_limit_factor=0.5, queue_upper_limit_factor=2.0. - -## 11. Settings / env vars (server/settings.py) - -- `DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED` → `SERVER_BACKGROUND_PROCESSING_DISABLED/ENABLED` (lines 47-50) — the only kill switch; disables BOTH scheduled and pipeline tasks (app.py:178). -- `DSTACK_SERVER_EXECUTOR_MAX_WORKERS` (line 52, default 128) — thread pool for `run_async`. -- `DSTACK_SERVER_MAX_OFFERS_TRIED` (line 54, default 25). -- `DSTACK_DB_POOL_SIZE` (44, default 20) / `DSTACK_DB_MAX_OVERFLOW` (45, default 20). -- `DSTACK_ENABLE_PROMETHEUS_METRICS` gates the prometheus scheduled tasks. -There are NO env vars for per-pipeline workers/intervals/lock timeouts — constructor defaults only. - -## 12. Scheduled-task skeleton (if endpoints ever needed one) - -`process_idle_volumes` (scheduled_tasks/idle_volumes.py:24-63): decorated `@sentry_utils.instrument_scheduled_task`; gets `(lock, lockset) = get_locker(get_db().dialect_name).get_lockset(VolumeModel.__tablename__)`; under `async with lock:` selects candidate ids with `.where(..., VolumeModel.lock_expires_at.is_(None), VolumeModel.id.not_in(lockset)).limit(10).with_for_update(skip_locked=True, key_share=True)` and adds ids to the lockset; processes; `finally: lockset.difference_update(volume_ids)`. Note it defers real work to the pipeline by setting `to_be_deleted=True`. The `lock_expires_at.is_(None)` check is how a scheduled task avoids touching rows a pipeline currently holds. - -## 13. Precise recipe for a `process_endpoints` pipeline task - -1. **Model**: `class EndpointModel(PipelineModelMixin, BaseModel)` in `src/dstack/_internal/server/models.py` with `id UUIDType(binary=False) pk default uuid4`, `created_at`, `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime)` (init to created_at on insert), `status` (use `EnumAsString(EndpointStatus, 100)`), `status_message`, `deleted: Mapped[bool]`, FKs to projects/users, and `__table_args__ = (Index("ix_endpoints_pipeline_fetch_q", last_processed_at.asc(), postgresql_where=, sqlite_where=),)`. -2. **Migration**: alembic revision under `src/dstack/_internal/server/migrations/versions/2026/`, modeled on `857d8fa7fcc5` (or a plain create_table since it's a new table). -3. **Pipeline**: `src/dstack/_internal/server/background/pipeline_tasks/endpoints.py` with `EndpointPipelineItem(PipelineItem)` (+ `status` field), `EndpointPipeline(Pipeline[EndpointPipelineItem])` (copy VolumePipeline verbatim: __init__ wires `Heartbeater(model_type=EndpointModel, ...)`, `EndpointFetcher`, N× `EndpointWorker`; `hint_fetch_model_name -> EndpointModel.__name__`; expose `_heartbeater/_fetcher/_workers` via private-name properties), `EndpointFetcher.fetch` copying the section-4 query (status filter e.g. `status.in_([SUBMITTED, PROVISIONING, ...])`, `lock_owner IS NULL OR == EndpointPipeline.__name__`), `EndpointWorker.process` doing refetch-by-token → per-status dispatch → apply-with-token pattern. Suggested tuning for agent-driven provisioning: workers_num ~10, min_processing_interval 5-15s, lock_timeout 30s, heartbeat_trigger 15s (heartbeater makes multi-minute steps safe, but prefer chunking into statuses per section 9.2). -4. **Register** in `PipelineManager.__init__` builtin list (`background/pipeline_tasks/__init__.py:35-48`). -5. **Submit-side hint**: in the endpoint-create service function, accept `pipeline_hinter: PipelineHinterProtocol` (injected in the router via `Depends(get_pipeline_hinter)`, as `services/volumes.py` does) and call `pipeline_hinter.hint_fetch(EndpointModel.__name__)` after commit. -6. **Sub-resource**: if the endpoint pipeline must mutate its RunModel/service, follow the `_lock_related_jobs` pattern (claim child rows with same lock_token + lock_owner=EndpointPipeline.__name__, reset-own-lock-for-retry if children unavailable) — or better, avoid cross-pipeline row writes and interact with runs via the `services/runs` submit/terminate service functions the way API handlers do (cross-pipeline direct writes are only done for the benign `skip_min_processing_interval` flag, jobs_running.py:990-999). -7. **Tests**: `src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py`, mirroring `test_volumes.py`: fixtures constructing `EndpointFetcher(queue=asyncio.Queue(), queue_desired_minsize=1, ..., heartbeater=Mock())` and `EndpointWorker(queue=Mock(), heartbeater=Mock(), pipeline_hinter=Mock())`, `@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)`, helpers from `dstack._internal.server.testing.common`. - -## Gotchas -1. STALE KNOWLEDGE TRAP: the old `process_runs.py`-style `@sentry_utils.instrument_background_task` + apscheduler-per-N-seconds processing modules do NOT exist anymore; runs/jobs/fleets/instances/volumes/gateways are all fetch/worker Pipelines under background/pipeline_tasks/. Also `get_locker()` now REQUIRES a dialect_name arg: `get_locker(get_db().dialect_name)`. -2. The in-memory lockset locker is NOT the multi-replica mechanism — on Postgres it's a no-op DummyResourceLocker by design. Cross-replica exclusion = lock_expires_at/lock_token/lock_owner columns + FOR UPDATE SKIP LOCKED in the fetch + token-guarded UPDATEs. Any plan that says "acquire the locker to be replica-safe" is wrong. -3. Every write after fetch MUST carry `WHERE lock_token == item.lock_token` and every terminal apply MUST both unlock (set_unlock_update_map_fields) and bump last_processed_at (set_processed_update_map_fields) in the SAME UPDATE; forgetting last_processed_at causes hot-looping, forgetting unlock stalls the row for lock_timeout. -4. `min_processing_interval` gating uses `last_processed_at == created_at` (or `== submitted_at` for runs) to fast-path brand-new rows — so the row-creation code must set last_processed_at equal to created_at, not leave it NULL (column is non-nullable). -5. APScheduler `max_instances=1` and DateTrigger init tasks are per-replica; scheduled tasks run concurrently on every replica and must be idempotent. Don't put endpoint provisioning there — use a pipeline (that's exactly what the two docstrings at pipeline_tasks/__init__.py:107-109 and scheduled_tasks/__init__.py:38-41 distinguish). -6. `hint_fetch` only wakes the local replica's fetcher; without it processing still happens within a few seconds via the polling fetch delays (max ~5s + jitter), so it's an optimization, not a correctness requirement. -7. Worker `process()` exceptions are swallowed+logged by Worker.start(); the row then stays locked until lock_expires_at passes (~lock_timeout retry latency). If a replica dies mid-process, work is re-picked from scratch after lock expiry — so the endpoint agent/deploy flow must be idempotent or persisted as a status state machine (see instances PENDING→PROVISIONING→deadline-checked pattern) rather than a single monolithic process() that runs for many minutes; the heartbeater technically allows unbounded process() duration, but crash-recovery restarts the whole step. -8. `run_async` uses the loop's default executor (ThreadPoolExecutor, 128 threads, app.py:123); long blocking calls (LLM API calls, SSH) should go through it or a native-async client, or they'll block heartbeats for the entire replica. -9. DB sessions: `get_session_ctx()` commits on clean exit, `expire_on_commit=False`, `autoflush=False` — mutations made on ORM objects during fetch are only persisted because fetch explicitly commits; in workers prefer explicit `update()` statements over ORM attribute mutation. -10. `prev_lock_expired` on PipelineItem is set by all fetchers but consumed nowhere — don't build logic assuming it does something. -11. `lock_owner` is a pipeline-name namespace: fetchers only take rows where lock_owner is NULL or their own class name, and the reset-for-retry path deliberately KEEPS lock_owner while nulling lock_expires_at/lock_token. If two subsystems (e.g. an endpoints pipeline and the run pipeline) can lock the same table, both must honor this filter. -12. Datetimes are stored via NaiveDateTime columns while `get_current_datetime()` returns tz-aware UTC — copy existing comparison patterns verbatim rather than mixing aware/naive by hand. -13. Background processing can be disabled entirely (DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED, used in tests); services must not assume the pipeline manager exists — `get_pipeline_hinter` already handles this with a no-op hinter. diff --git a/endpoint-implementation-research/claude-agent-sdk.md b/endpoint-implementation-research/claude-agent-sdk.md deleted file mode 100644 index 5e39a4a6c7..0000000000 --- a/endpoint-implementation-research/claude-agent-sdk.md +++ /dev/null @@ -1,407 +0,0 @@ -# claude-agent-sdk - -## Summary -Comparison of Claude Agent SDK (Python) vs. plain anthropic SDK for embedding an autonomous deployment agent in dstack. **Recommend Agent SDK (v0.2.110, Python 3.10+) for first implementation**: it bundles the Claude Code CLI binary (not Node.js), provides built-in autonomous tool execution with permission control modes, handles the agentic loop automatically, and supports skills loading convention. The plain anthropic SDK (v0.116.0, Python 3.9+) is lighter-weight but requires manual tool-loop implementation. Both authenticate via ANTHROPIC_API_KEY. Agent SDK deployment footprint: bundled binary CLI + Python runtime ~300MB per platform. Defer plain SDK approach until Agent SDK proves insufficient for the autonomous deployment use case. - -## Key files -- claude-agent-sdk (PyPI: pip install claude-agent-sdk) — query(), ClaudeSDKClient, ClaudeAgentOptions, @tool, create_sdk_mcp_server(), HookMatcher, AgentDefinition, AsyncAnthropic pattern — Official Python Agent SDK. Version 0.2.110 released June 24, 2026. Requires Python 3.10+. Bundles Claude Code CLI binary (not Node.js; packed as platform-specific wheels for macOS ARM64/x86-64, Linux, Windows). GitHub: anthropics/claude-agent-sdk-python. Bundling mechanism: CLI is included in wheel distribution; used by default via stdio; custom path override supported via ClaudeAgentOptions(cli_path=...). -- anthropic (PyPI: pip install anthropic) — Anthropic, AsyncAnthropic, messages.create(), messages.stream(), @beta_tool, tool_runner, messages.count_tokens(), messages.batches, ToolUseBlock, ToolResultBlock — Official Anthropic Python SDK. Version 0.116.0 released July 2, 2026. Requires Python 3.9+. Pure Python, no external binaries. GitHub: anthropics/anthropic-sdk-python. Includes optional extras: [bedrock], [vertex], [aws] for cloud integrations; [aiohttp] for better async concurrency. -- https://code.claude.com/docs/en/agent-sdk/overview — Built-in tools, hook callbacks, permission_mode values, mcp_servers dict config, agents dict, allowed_tools/disallowed_tools lists — Official Agent SDK documentation. Covers query()/ClaudeSDKClient interfaces, built-in tools (Read, Write, Edit, Bash, Monitor, Glob, Grep, WebSearch, WebFetch, AskUserQuestion), hooks lifecycle (PreToolUse, PostToolUse, Stop, SessionStart, SessionEnd, UserPromptSubmit), subagents, MCP integration, permission model, sessions with resume(). Skills convention documented: .claude/skills/*/SKILL.md auto-loaded from working directory and ~/.claude/. - -## Details -## A) Claude Agent SDK (Python) - -### Package & Installation -- **Name**: `claude-agent-sdk` -- **Version**: 0.2.110 (June 24, 2026) -- **Python**: 3.10+ -- **Installation**: `pip install claude-agent-sdk` -- **Bundling**: Bundles Claude Code CLI **as a binary** (not Node.js). Platform-specific wheels (macOS ARM64/x86-64, Linux, Windows). No separate Node.js installation or subprocess complexity. - -### Core APIs - -**Simple interface:** -```python -from claude_agent_sdk import query, ClaudeAgentOptions - -async for message in query( - prompt="Deploy model X as dstack service", - options=ClaudeAgentOptions(allowed_tools=["Bash", "Read", "Write"]) -): - print(message) -``` - -**Interactive sessions:** -```python -from claude_agent_sdk import ClaudeSDKClient - -async with ClaudeSDKClient(options=options) as client: - await client.query(prompt) - async for msg in client.receive_response(): - print(msg) -``` - -### Custom Tool Definition -Via `@tool` decorator + in-process MCP server (no subprocess overhead): -```python -from claude_agent_sdk import tool, create_sdk_mcp_server - -@tool("verify_endpoint", "Verify endpoint reachability", {"url": str}) -async def verify_endpoint(args): - # Direct Python execution - return {"content": [{"type": "text", "text": f"OK: {args['url']}"}]} - -server = create_sdk_mcp_server( - name="dstack-tools", - version="1.0.0", - tools=[verify_endpoint] -) - -options = ClaudeAgentOptions( - mcp_servers={"tools": server}, - allowed_tools=["mcp__tools__verify_endpoint"] # Pre-approve -) -``` - -### Built-In Tools (Pre-Integrated) -Read, Write, Edit, Bash, Monitor, Glob, Grep, WebSearch, WebFetch, AskUserQuestion. All available without configuration. - -### Permission Model (for Autonomous Headless Operation) -```python -options = ClaudeAgentOptions( - allowed_tools=["Read", "Write", "Bash"], # Auto-approve list - disallowed_tools=["DangerousTool"], # Block list - permission_mode='acceptEdits' # Auto-accept file edits -) -``` -Evaluation order: `allowed_tools` → `disallowed_tools` → `permission_mode` fallback. No interactive approval prompts when tools pre-approved. **Can run fully autonomous in headless Docker environment.** - -### Hooks (for Observability & Control) -```python -async def log_exec(input_data, tool_use_id, context): - action = input_data.get("tool_input", {}) - audit_log.write(f"{tool_use_id}: {action}\n") - return {} - -options = ClaudeAgentOptions( - hooks={ - "PostToolUse": [ - HookMatcher(matcher="Bash", hooks=[log_exec]) - ] - } -) -``` - -### Authentication & Model Selection -- **API Key**: `ANTHROPIC_API_KEY` environment variable (standard) -- **Models**: Supports Claude Opus 4.8, Sonnet 4.6, Haiku 4.5, Fable 5 (June 2026 launch), Mythos 5 (trusted access) -- **Model config**: Set via options or defaults to latest available - -### Skills Loading -SDK supports `.claude/` filesystem convention: -- `.claude/skills/*/SKILL.md` – Auto-loaded skill definitions -- `.claude/CLAUDE.md` – Project memory/context -- `setting_sources` parameter to restrict which sources load - -**Not explicitly documented**: whether SDK has programmatic API to load skills from custom directories (e.g., `setting_sources` list-based config). This may require filesystem symlinks or copying skills into `.claude/`. - -### Sessions & Context Persistence -```python -# Capture session ID on init -session_id = message.data["session_id"] - -# Resume later with full context -async for msg in query(prompt="...", options=ClaudeAgentOptions(resume=session_id)): - ... -``` - -### Deployment Footprint -- **Per-platform binary**: ~100–150 MB (CLI binary alone) -- **SDK library**: ~50 MB -- **Python runtime**: depends on container base -- **No Node.js required**: Binary CLI, not JavaScript runtime -- **Subprocess management**: Minimal; SDK handles stdio communication with bundled binary -- **Concurrent sessions**: Each async query spawns a CLI process; manage concurrency via asyncio limits - -## B) Plain anthropic Python SDK - -### Package & Installation -- **Name**: `anthropic` -- **Version**: 0.116.0 (July 2, 2026) -- **Python**: 3.9+ -- **Installation**: `pip install anthropic` -- **Size**: ~10 MB, pure Python - -### Core API (Manual Tool-Use Loop) - -```python -from anthropic import Anthropic - -client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) - -messages = [{"role": "user", "content": "Deploy model X"}] -tools = [{"name": "bash", "description": "...", "input_schema": {...}}] - -while True: - response = client.messages.create( - model="claude-opus-4-8", - max_tokens=4096, - tools=tools, - messages=messages - ) - - if response.stop_reason == "tool_use": - # Manual loop: find tool calls, execute, collect results - for block in response.content: - if block.type == "tool_use": - result = execute_tool(block.name, block.input) - messages.append({"role": "assistant", "content": response.content}) - messages.append({ - "role": "user", - "content": [{ - "type": "tool_result", - "tool_use_id": block.id, - "content": result - }] - }) - break - else: - break # stop_reason == "end_turn" - -print(response.content[-1].text) -``` - -### Tool Definition (Helper Decorator) -```python -from anthropic import beta_tool - -@beta_tool -def bash_exec(command: str) -> str: - """Execute bash command. Args: command (str): bash to run""" - return subprocess.check_output(command, shell=True, text=True) - -@beta_tool -def verify_endpoint(url: str) -> str: - """Verify endpoint. Args: url (str): endpoint URL""" - import requests - return "OK" if requests.head(url).ok else "FAIL" -``` - -The `@beta_tool` decorator auto-generates the tool schema from function signature and docstring. **No built-in permission control**; you control tool availability by including/excluding from the `tools` list. - -### Tool Runner Helper (Simplifies Loop) -```python -from anthropic import beta_tool - -runner = client.beta.messages.tool_runner( - model="claude-opus-4-8", - max_tokens=4096, - tools=[bash_exec, verify_endpoint], - messages=[{"role": "user", "content": "Deploy model X"}] -) - -for message in runner: - if hasattr(message, "content"): - print(message.content) -``` -Still requires you to implement tool execution; the runner just collects the loop boilerplate. **Not equivalent to Agent SDK's autonomous execution.** - -### Streaming -```python -with client.messages.stream( - model="claude-opus-4-8", - max_tokens=4096, - tools=tools, - messages=messages -) as stream: - for text in stream.text_stream: - print(text, end="", flush=True) - final = stream.get_final_message() -``` - -### Web Search & Web Fetch Tools -Both available as built-in tools (since April 2026): -- `web_search_20260318` (dynamic filtering variant) – $10 per 1,000 searches + standard token costs -- `web_fetch_20250305` – Included in standard requests -Models supporting web tools: Opus 4.7+, Sonnet 4.6+, Opus 4.6+, Opus 4.5+, Haiku 4.5+, Sonnet 4.5+ - -### Prompt Caching -```python -response = client.messages.create( - model="claude-opus-4-8", - max_tokens=4096, - system=[ - { - "type": "text", - "text": "You are a deployment agent...", - "cache_control": {"type": "ephemeral"} - } - ], - messages=[...] -) -``` -Cached input costs 90% less. Useful for long system prompts. Works across multiple requests within 5-minute window. - -### Token Counting -```python -count = client.messages.count_tokens( - model="claude-opus-4-8", - messages=[{"role": "user", "content": "Hello"}], - tools=tools -) -print(count.input_tokens, count.output_tokens) - -# Post-request usage -response = client.messages.create(...) -print(response.usage.input_tokens, response.usage.output_tokens) -``` - -### Structured Outputs -```python -response = client.messages.create( - model="claude-opus-4-8", - max_tokens=4096, - messages=[...], - structured_output={"schema": {...}, "strict": True} -) -# Returns validated JSON matching schema -``` - -### Authentication & Model Selection -- **API Key**: `ANTHROPIC_API_KEY` environment variable -- **Models**: Same lineup as Agent SDK (Opus 4.8, Sonnet 4.6, Haiku 4.5, Fable 5, Mythos 5) -- **Cloud provider integrations**: Bedrock, Vertex AI, Foundry, Claude Platform on AWS (via `AnthropicBedrock`, `AnthropicVertex`, etc.) - -### Deployment Footprint -- **Pure Python**: ~10 MB + httpx dependency -- **No CLI binary**: No subprocess overhead -- **No Node.js**: No runtime bloat -- **Lightweight**: Ideal for resource-constrained containers - -### Error Handling & Reliability -```python -from anthropic import APIConnectionError, RateLimitError, APITimeoutError - -try: - response = client.messages.create(...) -except RateLimitError: - # Automatic retries (default 2x); configure with max_retries=N - pass -except APITimeoutError: - # Default 10-minute timeout; configure per-request or globally - pass -``` - ---- - -## Comparison Table - -| Feature | Agent SDK | Plain SDK | -|---------|-----------|-----------| -| **Package** | `claude-agent-sdk` 0.2.110 | `anthropic` 0.116.0 | -| **Python** | 3.10+ | 3.9+ | -| **Size** | ~300 MB (bundled binary CLI) | ~10 MB (pure Python) | -| **Node.js required** | No (binary CLI) | No | -| **Agentic loop** | Automatic (query/ClaudeSDKClient) | Manual (messages.create loop) | -| **Tool definition** | `@tool` + in-process MCP | `@beta_tool` decorator | -| **Permission control** | `allowed_tools`, `disallowed_tools`, `permission_mode` | None (list-based inclusion) | -| **Headless autonomous** | Yes (permission_mode='acceptEdits') | Requires manual implementation | -| **Built-in tools** | Read, Write, Edit, Bash, Monitor, Glob, Grep, WebSearch, WebFetch, AskUserQuestion | None (define custom or use tool schema) | -| **Streaming** | Automatic in query loop | Via messages.stream() | -| **Web search/fetch** | Included | Included (webSearch/webFetch tools, since Apr 2026) | -| **Prompt caching** | Not mentioned | Yes, cache_control parameter | -| **Structured outputs** | Not mentioned | Yes, structured_output parameter | -| **Token counting** | Pre-request via query context | messages.count_tokens() + response.usage | -| **Sessions** | Yes, resume=session_id | Manual message history management | -| **Skills loading** | `.claude/skills/*/SKILL.md` convention | N/A | -| **Hooks** | Pre/PostToolUse, SessionStart/End, UserPromptSubmit | N/A | -| **Deployment** | Subprocess (CLI binary via stdio) | In-process HTTP calls | -| **Concurrency** | asyncio-based, one CLI process per query | asyncio-based, HTTP connection pooling | - ---- - -## Recommendation: START WITH AGENT SDK - -### Rationale -1. **Autonomous operation**: `permission_mode='acceptEdits'` + `allowed_tools` = fully headless, no approval prompts. dstack agent runs in a Docker container without user interaction. -2. **Tool loop already solved**: Agent SDK handles the agentic loop internally. You write the objective ("deploy model X"), Agent SDK calls tools, parses responses, repeats until `stop_reason != "tool_use"`. -3. **Built-in tools**: Read, Write, Edit, Bash, WebSearch, WebFetch all pre-integrated; no schema boilerplate. -4. **Permission granularity**: Block dangerous tools, auto-approve safe ones—critical for production agents touching infra. -5. **Skills convention**: `.claude/skills/` can hold domain-specific prompt files (e.g., "dstack-deployment-best-practices.md") loaded automatically. -6. **Observability**: Hooks let you audit every tool call, useful for logging/compliance in a managed service. -7. **No Node.js surprise**: Binary CLI, not JavaScript; deployment burden is bundle size, not runtime complexity. - -### Gotchas / Verification Needed -- **Skill directory loading**: Documentation shows `.claude/skills/*/SKILL.md` convention but does NOT explicitly document programmatic API to point SDK at custom skill directories. May require filesystem setup or `setting_sources` config (not yet verified in official docs). -- **Prompt caching**: Agent SDK docs don't mention cache_control support; unclear if long system prompts benefit from caching. **May need to implement separately via custom MCP server or plain SDK for cost control on repeated deployments.** -- **Concurrent agents**: Each query spawns a CLI subprocess. In a multi-tenant dstack setup, concurrency management (asyncio semaphores) is on you. -- **Model selection**: Agent SDK supports model choice but default strategy unclear. Recommend explicit model selection for cost control (Haiku 4.5 for fast tasks, Sonnet 4.6 for complex deployments). -- **Structured outputs**: Agent SDK doesn't mention structured_output schema support. If the agent must return a validated JSON deployment report, may need to parse text or use plain SDK. - -### Suggested Implementation Pattern (dstack AgentService) - -```python -from claude_agent_sdk import query, ClaudeAgentOptions - -class ClaudeAgentService: - def __init__(self, model: str = "claude-sonnet-4-6"): - self.model = model - - async def deploy(self, objective: str) -> dict: - # Pre-approve safe tools for autonomous operation - options = ClaudeAgentOptions( - allowed_tools=["Read", "Write", "Bash", "WebSearch"], - disallowed_tools=["DeleteFile"], # Custom safety rules - permission_mode="acceptEdits", - system_prompt=f"You are a dstack deployment agent. Objective: {objective}" - ) - - result_text = "" - async for message in query(prompt=objective, options=options): - if hasattr(message, "result"): - result_text = message.result - - return {"success": True, "output": result_text} -``` - -### Defer to Plain SDK If -- Prompt caching becomes critical for cost (long system prompts on every deployment). Plain SDK's `cache_control` is explicit. -- Agent SDK's structured output support lags behind needs (plain SDK has `structured_output` parameter). -- Concurrent session management becomes a bottleneck (plain SDK's HTTP is more amenable to connection pooling). -- Custom tools become complex and in-process MCP not sufficient (plain SDK's tool schema is more flexible). - -### Cost / Token Considerations -- **Agent SDK**: No additional overhead; standard Claude pricing (input/output tokens). CLI binary overhead is one-time disk, not per-request. -- **Plain SDK**: Same; adds prompt caching option (10% cost on cached input). -- **Web search**: Both support. $10 per 1,000 searches. -- **Batch processing**: Not available in Agent SDK (built on CLI); plain SDK offers `messages.batches` for 50% discount on bulk async requests (might matter for large dstack deployments). - -### What to Implement First (Behind AgentService Abstraction) -1. Basic `deploy(objective: str) -> dict` method returning success + logs -2. Tool allowlist (Bash, Read, WebSearch) + blocklist (Delete, Exec) -3. Hook for audit logging of all tool calls -4. Error handling for API key missing (early catch) -5. Model selection flag (default Sonnet 4.6, option for Haiku 4.5 for speed) - -Defer: prompt caching, structured output schema, skill directory custom paths, concurrent session management—validate Agent SDK meets MVP first. - -## Gotchas -**Agent SDK:** -- CLI subprocess per query; no persistent connection pooling. Concurrency scales with asyncio but spawns new processes. In high-throughput scenarios (100+ concurrent deployments), process overhead may become visible. -- Skill directory loading documented as convention (`.claude/skills/*/SKILL.md`) but no explicit programmatic API to restrict sources or load from custom paths. May need filesystem manipulation to use custom skill directories in dstack. -- Prompt caching not mentioned in Agent SDK docs; if long system prompts are repeated per deployment, caching benefit may be missed. Plain SDK's `cache_control` is explicit. -- Structured outputs not mentioned; if deployment report must be guaranteed JSON schema, may need post-processing or custom tool returning JSON string. -- Model selection: Default behavior unclear. Recommend explicit model flag in options to control cost (Haiku 4.5 for fast, Sonnet 4.6 for complex). -- Permission modes work but are SDK-level. If dstack needs finer-grained control (e.g., Bash only in specific directories), hook-based filtering required. - -**Plain SDK:** -- Manual tool-use loop is boilerplate-heavy; easy to introduce bugs (forgetting to append assistant message, wrong tool_result format). Tool runner helper exists but does NOT automate loop—you still implement execution. -- No built-in permission model; must implement allowlist/blocklist at application layer. -- No native session resumption; message history is your responsibility. -- Skills concept doesn't exist; custom tools are pure Python functions or external MCP servers. -- Hooks don't exist; audit logging is manual. -- Lighter footprint but requires more application code to reach Agent SDK feature parity. - -**Both:** -- API key via env var (standard); no option to pass key at init time (security best practice for 12-factor), though both support explicit client = Anthropic(api_key=...). -- No built-in support for multi-replica coordination (e.g., one deployment agent per dstack replica). Session IDs are local; cross-replica session sharing requires external persistence layer. diff --git a/endpoint-implementation-research/config-models.md b/endpoint-implementation-research/config-models.md deleted file mode 100644 index 336b5dcf4b..0000000000 --- a/endpoint-implementation-research/config-models.md +++ /dev/null @@ -1,151 +0,0 @@ -# config-models - -## Summary -Mapped how dstack configuration types are defined and registered end-to-end. All top-level config types live in src/dstack/_internal/core/models/configurations.py, discriminated by a `type: Literal[...]` field, and are aggregated into four unions there (AnyRunConfiguration, AnyApplyConfiguration, BaseApplyConfiguration.__root__, AnyDstackConfiguration) plus an ApplyConfigurationType enum. CLI-side dispatch happens via apply_configurators_mapping in src/dstack/_internal/cli/services/configurators/__init__.py where each configurator class declares TYPE = ApplyConfigurationType.. There is NO generic server-side apply endpoint — each resource type (runs/fleets/gateways/volumes) has its own typed REST router registered in server/app.py and its own APIClient group in dstack/api/server/. No "endpoint" configuration type exists anywhere yet. ProfileParams (including `tags`) is mixed into run configurations only; fleets/gateways/volumes duplicate a subset of fields instead. - -## Key files -- src/dstack/_internal/core/models/configurations.py — RunConfigurationType, BaseRunConfiguration, DevEnvironmentConfiguration, TaskConfiguration, ServiceConfiguration, ServiceConfigurationParams, ReplicaGroup, ProbeConfig, ScalingSpec, RateLimit, AnyRunConfiguration, RunConfiguration, parse_run_configuration, ApplyConfigurationType, AnyApplyConfiguration, BaseApplyConfiguration, parse_apply_configuration, AnyDstackConfiguration, DstackConfiguration — THE registration hub: 4 unions + 1 enum must be touched for a new top-level type (lines 1366-1463). -- src/dstack/_internal/cli/services/configurators/__init__.py — apply_configurators_mapping, run_configurators_mapping, get_apply_configurator_class, load_apply_configuration — CLI dispatch table (lines 27-49) and YAML parse entrypoint (yaml.safe_load -> parse_apply_configuration, lines 62-86). -- src/dstack/_internal/cli/services/configurators/base.py — BaseApplyConfigurator (TYPE: ClassVar[ApplyConfigurationType], apply_configuration, delete_configuration, register_args), ApplyEnvVarsConfiguratorMixin — Abstract base every new configurator subclasses; env-var CLI args + EnvSentinel resolution from os.environ. -- src/dstack/_internal/core/models/profiles.py — ProfileParams (lines 310-493), ProfileParamsConfig, Profile, ProfileRetry, UtilizationPolicy, Schedule, SpotPolicy, CreationPolicy, parse_duration, parse_off_duration, parse_idle_duration — Complete ProfileParams field list incl. tags; mixed into run configs via multiple inheritance. -- src/dstack/_internal/core/models/envs.py — Env, EnvSentinel, EnvVarTuple — Env is a plain pydantic BaseModel with __root__ Union[List[str], Dict[str, Union[str, EnvSentinel]]] — deliberately NOT a CoreModel. -- src/dstack/_internal/core/models/common.py — CoreModel, CoreConfig, generate_dual_core_model, Duration, EntityReference, ApplyAction — pydantic-duality dual models: strict __request__ (extra=forbid) vs __response__ (extra=ignore). Custom Config must go through generate_dual_core_model. -- src/dstack/_internal/core/models/fleets.py — FleetConfiguration, CommonFleetConfigurationProps (type: Literal["fleet"]), BackendFleetConfiguraionProps, SSHFleetConfigurationProps, FleetSpec, Fleet, FleetPlan, ApplyFleetPlanInput, FleetStatus — Non-run config pattern: no ProfileParams inheritance; FleetSpec = configuration + profile + merged_profile. -- src/dstack/_internal/core/models/gateways.py — GatewayConfiguration (type: Literal["gateway"]), GatewaySpec, Gateway, GatewayPlan, ApplyGatewayPlanInput, GatewayStatus — Simplest non-run config: flat CoreModel, no env, no ProfileParams; SUBMITTED/PROVISIONING/RUNNING/FAILED status enum is the closest analog to the planned endpoint lifecycle. -- src/dstack/_internal/core/models/volumes.py — BaseVolumeConfiguration (type: Literal["volume"]), AnyVolumeConfiguration, VolumeConfiguration, parse_volume_configuration, VolumeSpec, Volume, VolumeStatus — Two-stage discriminated parse: `type` then `backend`; explains why BaseApplyConfiguration vs AnyApplyConfiguration differ. -- src/dstack/_internal/cli/services/configurators/gateway.py — GatewayConfigurator (TYPE = ApplyConfigurationType.GATEWAY) — Best template for an async-provisioned resource configurator: get_plan -> confirm -> apply_plan -> poll status until RUNNING/FAILED. -- src/dstack/_internal/cli/services/configurators/run.py — BaseRunConfigurator, TaskConfigurator(:665), DevEnvironmentConfigurator(:680), ServiceConfigurator(:717), interpolate_env(:390) — Run-config CLI flow; ${{ env.* }} interpolation happens CLI-side. -- src/dstack/_internal/cli/commands/apply.py — ApplyCommand — Generic; dispatches on configuration.type via get_apply_configurator_class (line 92). No per-type edits needed. -- src/dstack/_internal/cli/commands/delete.py — DeleteCommand — Same generic dispatch (line 38); requires delete_configuration on the configurator. -- src/dstack/_internal/core/models/services.py — AnyModel, ChatModel, OpenAIChatModel, TGIChatModel, BaseChatModel — The `model` field type of ServiceConfiguration; discriminated on `format`. -- src/dstack/_internal/core/models/runs.py — RunSpec (:522), configuration field (:575), _merged_profile (:590-607) — How AnyRunConfiguration + Profile are merged into merged_profile — the pattern an endpoint spec with ProfileParams should replicate. -- src/dstack/_internal/server/app.py — register_routes (:239-269) — Server-side router registration; a new resource type needs its own router here. -- src/dstack/api/server/__init__.py — APIClient properties: fleets, runs, gateways, volumes, ... (:76-146) — Low-level HTTP client groups (dstack/api/server/_*.py); a new resource needs a new group. -- scripts/docs/gen_schema_reference.py — generate_schema_reference, sub_schema_reference — mkdocs hook expanding '#SCHEMA# dotted.model.Path' directives in mkdocs/docs/reference/dstack.yml/*.md. -- .github/workflows/build-artifacts.yml — line 248: DstackConfiguration.schema_json() — CI generates configuration.json editor schema from DstackConfiguration — picks up a new type automatically once added to AnyDstackConfiguration. - -## Details -# Configuration types in dstack — ground truth (verified 2026-07-03, master @ 28ea5f86f) - -No `endpoint` configuration type, `EndpointConfiguration` class, or `ENDPOINT` enum member exists anywhere in the codebase. `grep` confirms. - -## 1. Run configurations — `src/dstack/_internal/core/models/configurations.py` (1463 lines) - -### Discriminator & enums -- `RunConfigurationType(str, Enum)` (:77-80): `DEV_ENVIRONMENT = "dev-environment"`, `TASK = "task"`, `SERVICE = "service"`. -- Every configuration class carries `type: Literal[""] = ""` used as the pydantic discriminator. - -### Base class -`class BaseRunConfiguration(CoreModel)` (:484) — `type: Literal["none"]` (overridden by subclasses). Fields: -- `name: Optional[str] = None`, `image: Optional[str] = None`, `user: Optional[str] = None`, `privileged: bool = False`, `entrypoint: Optional[str] = None`, `working_dir: Optional[str] = None`, `home_dir: str = "/root"` (deprecated), `registry_auth: Optional[RegistryAuth] = None`, `python: Optional[PythonVersion] = None`, `nvcc: Optional[bool] = None`, `single_branch: Optional[bool] = None`, `env: Env = Env()` (:540-543), `shell: Optional[str] = None`, `resources: ResourcesSpec = ResourcesSpec()` (:554-556), `priority: Optional[int]` (0..100), `volumes: List[MountPoint] = []`, `docker: Optional[bool] = None`, `repos: list[RepoSpec] = []`, `files: list[FilePathMapping] = []`, `setup: CommandsList = []` (deprecated). -- Validators coerce strings: `volumes` via `parse_mount_point`, `files` via `FilePathMapping.parse`, `repos` via `RepoSpec.parse`; mutual exclusions for image/python/docker/nvcc. - -### Mixin param classes -- `ConfigurationWithPortsParams` (:657): `ports: List[Union[ValidPort, constr(regex=...), PortMapping]] = []` (`ValidPort = conint(gt=0, le=65536)` :53). -- `ConfigurationWithCommandsParams` (:672): `commands: CommandsList = []` + root_validator requiring `commands` or `image` (skipped when `replicas` is a list). -- `DevEnvironmentConfigurationParams` (:687): `ide` (vscode/cursor/windsurf/zed), `version`, `init: CommandsList`, `inactivity_duration`. -- `TaskConfigurationParams` (:768): `nodes: int = 1 (ge=1)`. - -### Concrete classes (note MRO — ProfileParams first) -- `DevEnvironmentConfiguration(ProfileParams, BaseRunConfiguration, ConfigurationWithPortsParams, DevEnvironmentConfigurationParams, generate_dual_core_model(DevEnvironmentConfigurationConfig))` (:752) — `type: Literal["dev-environment"] = "dev-environment"` (:759). -- `TaskConfiguration(ProfileParams, BaseRunConfiguration, ConfigurationWithCommandsParams, ConfigurationWithPortsParams, TaskConfigurationParams, generate_dual_core_model(TaskConfigurationConfig))` (:782) — `type: Literal["task"] = "task"` (:790). -- `ServiceConfiguration(ProfileParams, BaseRunConfiguration, ConfigurationWithCommandsParams, ServiceConfigurationParams, generate_dual_core_model(ServiceConfigurationConfig))` (:1328) — `type: Literal["service"] = "service"` (:1335); property `replica_groups -> List[ReplicaGroup]` (:1337-1363). - -Each has a paired `*Config` class (e.g. `ServiceConfigurationConfig` :1316) that composes `schema_extra` from `ProfileParamsConfig`, `BaseRunConfigurationConfig`, `ServiceConfigurationParamsConfig` — needed because pydantic-duality requires Config passed via `generate_dual_core_model()`. - -### ServiceConfiguration own fields (`ServiceConfigurationParams` :961-1313) -- `port: Union[ValidPort, constr(regex=r"^[0-9]+:[0-9]+$"), PortMapping]` — REQUIRED; validator coerces int → `PortMapping(local_port=80, container_port=v)` (:1050-1056). -- `gateway: Optional[Union[bool, EntityReference, str]] = None` (str coerced to EntityReference). -- `strip_prefix: bool = True`. -- `model: Optional[AnyModel] = None` (:993-1003); validator `convert_model` (:1058-1062): str → `OpenAIChatModel(type="chat", name=v, format="openai")`. `AnyModel = Union[ChatModel]`; `ChatModel = Annotated[Union[TGIChatModel, OpenAIChatModel], Field(discriminator="format")]` (core/models/services.py:75-76). `OpenAIChatModel` has `prefix: str = "/v1"`. -- `https: Optional[Union[bool, Literal["auto"]]] = None`. -- `auth: bool = True`. -- `scaling: Optional[ScalingSpec] = None` (`ScalingSpec` :213 — metric Literal["rps"], target float, window, scale_up_delay/scale_down_delay Durations). -- `rate_limits: list[RateLimit] = []` (:282). -- `probes: Optional[list[ProbeConfig]] = None` (:1019-1026) — `None` = may get default when `model` set; `[]` = explicitly none. `ProbeConfig` (:365): `type: Literal["http"]`, `url` (default `/`), `method` (get/post/put/delete/patch/head), `headers: list[HTTPHeaderSpec]`, `body`, `timeout` (default 10s), `interval` (default 15s), `ready_after` (default 1), `until_ready` (default false). The default model probe is built server-side in `server/services/jobs/configurators/base.py::_openai_model_probe_spec` (:472-491): POST `{prefix}/chat/completions` with `{"model": name, "messages":[{"role":"user","content":"hi"}], "max_tokens":1}`, timeout `OPENAI_MODEL_PROBE_TIMEOUT = 30`. -- `replicas: Optional[Union[List[ReplicaGroup], Range[int]]] = None` (:1028-1040); `ReplicaGroup` (:817) has name, count: Range[int], scaling, resources, spot_policy, reservation, commands, image, python, nvcc, docker, privileged, router. Multiple root_validators enforce group vs top-level mutual exclusions (:1123-1313). -- `router: Optional[AnyServiceRouterConfig] = None` (from `core/models/routers.py`). -- `resources` (inherited): `ResourcesSpec` (core/models/resources.py:377) — cpu, memory, shm_size, gpu, disk. - -### Unions & parsers (the registration hub) -- `AnyRunConfiguration = Union[DevEnvironmentConfiguration, TaskConfiguration, ServiceConfiguration]` (:1366). -- `class RunConfiguration(CoreModel)` (:1369): `__root__: Annotated[AnyRunConfiguration, Field(discriminator="type")]`. -- `def parse_run_configuration(data: dict) -> AnyRunConfiguration` (:1376-1381) — raises `ConfigurationError` on ValidationError. -- `class ApplyConfigurationType(str, Enum)` (:1384-1390): DEV_ENVIRONMENT, TASK, SERVICE, FLEET, GATEWAY, VOLUME. -- `AnyApplyConfiguration = Union[AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, AnyVolumeConfiguration]` (:1393-1398). -- `class BaseApplyConfiguration(CoreModel)` (:1401-1421): `__root__: Annotated[Union[AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, BaseVolumeConfiguration], Field(discriminator="type")]` — note `BaseVolumeConfiguration` here (not the backend union) because volumes need a second parse on `backend`. -- `def parse_apply_configuration(data: dict) -> AnyApplyConfiguration` (:1424-1437): pass 1 with `BaseApplyConfiguration.__response__` (extra ignored) to find the type; if not a volume, pass 2 strict (`BaseApplyConfiguration.parse_obj`, extra forbidden) for validation, return pass-1 object; volumes delegate to `parse_volume_configuration` (volumes.py:191). -- `AnyDstackConfiguration = Union[AnyRunConfiguration, FleetConfiguration, GatewayConfiguration, VolumeConfiguration]` (:1440-1445) — uses the `VolumeConfiguration` ROOT model, not `AnyVolumeConfiguration`. -- `class DstackConfiguration(CoreModel)` (:1448-1463): root over `AnyDstackConfiguration` with draft-07 `$schema` + `additionalProperties: True` — used ONLY by CI (`.github/workflows/build-artifacts.yml:248`: `python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json())" > /tmp/json-schemas/configuration.json`). - -## 2. Non-run configuration models - -### Fleets — `core/models/fleets.py` -- `CommonFleetConfigurationProps` (:211): `type: Literal["fleet"] = "fleet"`, `name`, `placement`, `blocks`. -- `BackendFleetConfiguraionProps` (:232 — NOTE the typo "Configuraion" is the real class name): nodes (`FleetNodesSpec`), reservation, resources, backends, regions, availability_zones, instance_types, spot_policy, retry, max_price, idle_duration, `tags: Optional[Dict[str,str]]` (:298), backend_options. These duplicate ProfileParams-ish fields — fleets do NOT inherit ProfileParams. -- `SSHFleetConfigurationProps` (:345): ssh_config, `env: Env = Env()`. -- `FleetConfiguration(SSHFleetConfigurationProps, BackendFleetConfiguraionProps, CommonFleetConfigurationProps, generate_dual_core_model(FleetConfigurationConfig))` (:362). -- `FleetSpec` (:393): `configuration: FleetConfiguration`, `configuration_path: Optional[str]`, `profile: Profile`, `merged_profile: Annotated[Profile, Field(exclude=True)]` computed by root_validator `_merged_profile` (:408-424) — loops `for key in ProfileParams.__fields__` and overrides profile values with non-None config values. -- Runtime model `Fleet` (:427): id, name, project_name, spec, created_at, `status: FleetStatus` (SUBMITTED/ACTIVE/TERMINATING/TERMINATED/FAILED :34-41 — comment says SUBMITTED/FAILED reserved for async processing), status_message, instances. Plan models: `FleetPlan` (:438), `ApplyFleetPlanInput` (:456). - -### Gateways — `core/models/gateways.py` -- `GatewayConfiguration(CoreModel)` (:59): `type: Literal["gateway"] = "gateway"`, name, default, `backend: BackendType` (required), `region: str` (required), instance_type, router, domain, public_ip, certificate, replicas, `tags` (:111). Flat class — no ProfileParams, no env, no resources. -- `GatewaySpec` (:125): configuration + configuration_path only. -- `GatewayStatus` (:17): SUBMITTED/PROVISIONING/RUNNING/FAILED — closest analog for an async-provisioned endpoint lifecycle. Runtime `Gateway` (:141) has status, status_message, created_at. -- `GatewayPlan` (:176), `ApplyGatewayPlanInput` (:185). - -### Volumes — `core/models/volumes.py` -- `BaseVolumeConfiguration(CoreModel)` (:36): `type: Literal["volume"] = "volume"`, `backend: Any` (overridden per subclass with `Literal[BackendType.X]`), name, size, auto_cleanup_duration, `tags` (:59). -- Backend subclasses: `AWSVolumeConfiguration` (:113), `GCPVolumeConfiguration` (:121), `RunpodVolumeConfiguration` (:129), `KubernetesVolumeConfiguration` (:137); `AnyVolumeConfiguration` union (:179); `VolumeConfiguration` root model discriminated on `backend` (:187); `parse_volume_configuration` (:191). -- `VolumeStatus` (:19): SUBMITTED/PROVISIONING/ACTIVE/FAILED with `finished_statuses()`. -- `VolumeSpec` (:198): configuration + configuration_path. - -### Key difference from run configs -Non-run configs have `name` for the resource itself, plain status enums, `Spec` wrapper `{configuration, configuration_path[, profile]}`, `Plan`, and `ApplyPlanInput{spec, current_resource}` models. Server exposes per-type REST endpoints; CLI configurators call `self.api.client..get_plan/apply_plan/delete` and poll. - -## 3. ProfileParams — `core/models/profiles.py:310-493` (complete) -`backends: Optional[List[BackendType]]`, `regions: Optional[List[str]]`, `availability_zones: Optional[List[str]]`, `instance_types: Optional[List[str]]`, `reservation: Optional[str]`, `spot_policy: Optional[SpotPolicy]` (spot/on-demand/auto), `retry: Optional[Union[ProfileRetry, bool]]`, `max_duration: Optional[Union[Literal["off"], int]]`, `stop_duration: Optional[Union[Literal["off"], int]]`, `max_price: Optional[float] (gt=0)`, `creation_policy: Optional[CreationPolicy]` (reuse/reuse-or-create), `idle_duration: Optional[int]`, `utilization_policy: Optional[UtilizationPolicy]`, `startup_order: Optional[StartupOrder]`, `stop_criteria: Optional[StopCriteria]`, `schedule: Optional[Schedule]` (cron), `fleets: Optional[list[Union[EntityReference, str]]]`, `instances: Optional[List[InstanceSelector]] (min_items=1)`, `tags: Optional[Dict[str, str]]` (:462-471, validated by `tags_validator`), `backend_options: Optional[List[AnyBackendProfileOptions]]`. All Optional/None-default. Duration validators pre-parse "2h"/"off" strings. `Profile = ProfileProps(name, default) + ProfileParams` (:514). `ProfileParamsConfig.schema_extra` (:289) adds string/bool alt types for max_duration/stop_duration/idle_duration/instances. - -## 4. Env — `core/models/envs.py` -`class Env(BaseModel)` (:42) — custom root model: `__root__: Union[List[str matching ^([a-zA-Z_][a-zA-Z0-9_]*)(=.*$|$)], Dict[str, Union[str, EnvSentinel]]] = {}`. Docstring explicitly: NOT a CoreModel because pydantic-duality doesn't play well with custom root models. List form normalized to dict by validator; bare `VAR` becomes `EnvSentinel(key=VAR)`. API: `as_dict()` (raises ValueError if sentinels unresolved), `update()`, `keys/values/items`, `__getitem__/__setitem__`, dict-like `copy()`. Sentinel resolution from `os.environ` happens CLI-side in `ApplyEnvVarsConfiguratorMixin.apply_env_vars` (cli/services/configurators/base.py:95-103); `-e KEY[=VALUE]` args registered by `register_env_args` (:83-93). - -## 5. Tags -Yes, supported: `ProfileParams.tags` gives all run configurations tags for free; fleets (fleets.py:298), gateways (gateways.py:111), volumes (volumes.py:59) declare their own. Validation: `src/dstack/_internal/utils/tags.py::tags_validator` — key `^[_\-a-zA-Z0-9]{1,60}$`, value `^[a-zA-Z0-9 .:/=_\-+@]{0,256}$`. No other "metadata" field exists on configurations. - -## 6. YAML parsing/validation flow for `dstack apply` -1. `ApplyCommand._command` (cli/commands/apply.py:72) → `load_apply_configuration(args.configuration_file)` (cli/services/configurators/__init__.py:62-86): resolves `$PWD/.dstack.yml`/`.yaml` or `-f FILE` or stdin (`-`), `yaml.safe_load`, then `parse_apply_configuration(dict)`. -2. `get_apply_configurator_class(configuration.type)` (:52-55) — `ApplyConfigurationType(configurator_type)` then dict lookup in `apply_configurators_mapping` (:27-39, built from class list `[DevEnvironmentConfigurator, TaskConfigurator, ServiceConfigurator, FleetConfigurator, GatewayConfigurator, VolumeConfigurator]` keyed by `cls.TYPE`). -3. Configurator instantiated with `api_client=self.api` (`dstack.api._public.Client`), `configurator.get_parser()` parses per-type extra args, then `apply_configuration(conf, configuration_path, command_args, configurator_args)`. -4. `dstack delete/destroy` (cli/commands/delete.py:35-44) uses the same dispatch and calls `delete_configuration`. -5. `dstack apply -h TYPE` (apply.py:27-36) parses TYPE via the `ApplyConfigurationType` enum. -6. Run-config interpolation: `BaseRunConfigurator.interpolate_env` (run.py:390-408) uses `VariablesInterpolator({"env": ...}, skip=["secrets"])` on registry_auth and probe fields. - -Server side: there is NO generic apply/config endpoint. Runs: client sends `RunSpec` (core/models/runs.py:522; `configuration: Annotated[AnyRunConfiguration, Field(discriminator="type")]` :575; `merged_profile` root_validator :590-607) to `runs` router. Fleets/gateways/volumes have their own typed routers. Routers registered in `server/app.py::register_routes` (:239-269): server, users, auth, projects, backends, fleets, instances, repos, runs, gpus, metrics, logs, secrets, gateways, volumes, service_proxy, model_proxy, prometheus, files, events, templates, exports, imports, sshproxy, public_keys. Low-level client groups: `dstack/api/server/_*.py` with properties on `APIClient` (api/server/__init__.py:76-146). Run-type-only server dispatch: `server/services/jobs/__init__.py::_get_job_configurator` (:316-333) maps `RunConfigurationType` → `{DevEnvironmentJobConfigurator, TaskJobConfigurator, ServiceJobConfigurator}` (server/services/jobs/configurators/{dev,task,service}.py each with `TYPE: RunConfigurationType`). - -## 7. EXHAUSTIVE checklist to add a new top-level config type `endpoint` - -Core models: -1. `src/dstack/_internal/core/models/configurations.py` — define `EndpointConfiguration` (with `type: Literal["endpoint"] = "endpoint"`); add `ENDPOINT = "endpoint"` to `ApplyConfigurationType` (:1384); add the class to `AnyApplyConfiguration` (:1393), to `BaseApplyConfiguration.__root__` union (:1411), and to `AnyDstackConfiguration` (:1440). (Or define the model in a new `core/models/endpoints.py` and import it, like fleets/gateways/volumes do — beware circular imports: configurations.py already imports fleets/gateways/volumes/profiles.) -2. If it's a standalone resource (like gateway/volume): also define `EndpointSpec {configuration, configuration_path}`, `Endpoint` (runtime: id, name, project_name, created_at, status, status_message, ...), `EndpointStatus` enum (mirror `GatewayStatus`: SUBMITTED/PROVISIONING/RUNNING(ACTIVE)/FAILED), `EndpointPlan`, `ApplyEndpointPlanInput`. - -CLI: -3. New `src/dstack/_internal/cli/services/configurators/endpoint.py`: `class EndpointConfigurator(ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator[EndpointConfiguration])` with `TYPE = ApplyConfigurationType.ENDPOINT`, implementing `apply_configuration()` and `delete_configuration()` (both abstract), optionally `register_args()`/`apply_args()`. GatewayConfigurator (gateway.py:37) is the closest template (async provisioning + status polling). -4. `src/dstack/_internal/cli/services/configurators/__init__.py` — add `EndpointConfigurator` to the class list building `apply_configurators_mapping` (:30-39). Do NOT add to `run_configurators_mapping` (run types only). `dstack apply`/`dstack delete`/`dstack apply -h endpoint` then work with no further CLI edits. - -Server (endpoint-as-own-resource path): -5. New router `src/dstack/_internal/server/routers/endpoints.py` + registration in `server/app.py::register_routes` (:239). -6. New `src/dstack/api/server/_endpoints.py` APIClient group + property in `src/dstack/api/server/__init__.py` (pattern :96-131), so the CLI configurator can call `self.api.client.endpoints.*`. -7. (Server services/DB/background processing are separate topics — not covered here.) - -Docs/schema: -8. `mkdocs/docs/reference/dstack.yml/endpoint.md` with `#SCHEMA# dstack._internal.core.models....EndpointConfiguration` directives (processed by `scripts/docs/gen_schema_reference.py`). -9. `mkdocs.yml` nav (:343-349) — add `- endpoint: docs/reference/dstack.yml/endpoint.md`. -10. `mkdocs/docs/reference/dstack.yml.md` index — add link to the new page. -11. Editor JSON schema (`.github/workflows/build-artifacts.yml:248`) regenerates automatically from `DstackConfiguration` once step 1 includes the type in `AnyDstackConfiguration`. - -Optional: -12. `src/dstack/api/__init__.py` — public alias (pattern: `Service = _ServiceConfiguration` :28-30). -13. If ProfileParams is inherited, also create `EndpointConfigurationConfig(ProfileParamsConfig)` composing `schema_extra` (pattern: `ServiceConfigurationConfig` :1316-1325) and pass it via `generate_dual_core_model(...)` as the last base class. - -## Gotchas -1) Pydantic v1 syntax throughout (`root_validator`/`validator`, `__root__` models, `constr(regex=...)`) — do not write v2-style code. 2) pydantic-duality: every CoreModel is dual (`__request__` extra=forbid / `__response__` extra=ignore). NEVER define `class Config` directly on a model — it breaks `__response__`; pass a custom Config class via `generate_dual_core_model(MyConfig)` as a BASE CLASS (last in MRO), and when combining mixins, the Config class must manually chain each parent's `schema_extra` (see ServiceConfigurationConfig at configurations.py:1316). 3) `BaseRunConfiguration.type` is `Literal["none"]` with NO default — every concrete subclass overrides it with its own Literal + default; the discriminator for apply configs is `type`, and volumes have a SECOND discriminator `backend` requiring the two-stage `parse_apply_configuration` (first pass parses with `__response__` i.e. extras ignored, second strict pass validates). A new type with only `type` as discriminator goes in the "Final configurations" part of `BaseApplyConfiguration.__root__`. 4) `AnyDstackConfiguration` (editor JSON schema union) uses the wrapped `VolumeConfiguration` root model, not `AnyVolumeConfiguration` — subtle asymmetry with `AnyApplyConfiguration`. 5) MRO matters: run configs list `ProfileParams` FIRST (`ServiceConfiguration(ProfileParams, BaseRunConfiguration, ...)`); ProfileParams already contains `tags`, `schedule`, `fleets`, `idle_duration` etc., so an endpoint config inheriting ProfileParams gets all of those including tags — no need to re-declare. 6) Fleet/gateway/volume configs do NOT inherit ProfileParams (fleets duplicate ~12 of its fields in `BackendFleetConfiguraionProps` — note the real class name has the typo "Configuraion"); only run configs use the merged_profile pattern (`RunSpec._merged_profile` runs.py:590 iterates `ProfileParams.__fields__`). 7) `Env` is NOT a CoreModel (plain BaseModel with custom root) and `EnvSentinel` values (bare `VAR` entries) are resolved from os.environ CLI-side only (`ApplyEnvVarsConfiguratorMixin.apply_env_vars`); a server-side agent creating configs from an endpoint's env must handle/forbid unresolved sentinels itself (`Env.as_dict()` raises ValueError on unresolved). 8) `get_apply_configurator_class` does `ApplyConfigurationType(configurator_type)` — forgetting the enum member makes `dstack apply` crash with ValueError before any useful message. 9) `ServiceConfiguration.port` is REQUIRED (no default) and coerced to `PortMapping(local_port=80, ...)`; `model: str` is coerced to `OpenAIChatModel(type="chat", format="openai", prefix="/v1")` — a preset/agent-generated service config must set `port` and should set `model` for the OpenAI-compatible proxy + default `/v1/chat/completions` probe (probe default is applied server-side in `server/services/jobs/configurators/base.py::_openai_model_probe_spec`, not in the model). 10) Old-style server background tasks were reworked: `server/background/pipeline_tasks` + `server/background/scheduled_tasks` (see app.py:25-27 imports `start_pipeline_tasks`, `start_scheduled_tasks`) — do not reference a `background/tasks` module. 11) The CLI never sends `AnyApplyConfiguration` to the server; each type maps to its own typed REST API (RunSpec/FleetSpec/GatewaySpec/VolumeSpec + plan/apply-plan endpoints), so a new endpoint type needs its own router + API client group, not a hook into an existing generic endpoint. 12) `dstack delete` requires `delete_configuration` implemented (abstract), and gateway/volume configurators poll async deletion — deletion of the new resource should follow the same async pattern. diff --git a/endpoint-implementation-research/critique.md b/endpoint-implementation-research/critique.md deleted file mode 100644 index f322f71353..0000000000 --- a/endpoint-implementation-research/critique.md +++ /dev/null @@ -1,63 +0,0 @@ -# Completeness critique - -## Gaps -### How is an endpoint named and how does its name map to a run name? Verified: ProfileParams (src/dstack/_internal/core/models/profiles.py:280-470) has NO `name` field, so the proposed config {type, model, env, ProfileParams} has no identity; run_name regex is ^[a-z][a-z0-9-]{1,40}$ while model ids are like `Qwen/Qwen3-32B` (uppercase, slash). Need: explicit `name` field on EndpointConfiguration (copy BaseRunConfiguration.name or FleetConfiguration.name pattern), a deterministic endpoint->run_name derivation/sanitization, and per-project uniqueness semantics (unique constraint vs volumes-style advisory-lock dance). -- Why: Without a naming scheme the plan cannot define the DB unique constraint, the endpoint->run linkage, or CLI apply/delete addressing; and submit_run silently deletes finished runs with a colliding run_name. -- Where: src/dstack/_internal/core/models/configurations.py (BaseRunConfiguration.name validator), src/dstack/_internal/core/models/fleets.py, src/dstack/_internal/server/services/volumes.py name-uniqueness dance - -### What is the concrete API for preset-vs-fleet matching, i.e. 'does an existing fleet have capacity for this preset'? Candidates verified to exist but not researched: services/runs/plan.py get_plan path (fleet-aware: combine_fleet_and_run_profiles, combine_fleet_and_run_requirements, check_can_create_new_cloud_instance_in_fleet at plan.py:37-41,64-65), get_offers_by_requirements(project, profile, requirements, ...) (services/offers.py:30), get_pool_instances (services/instances.py:723). Unknown: whether RunPlan/JobPlan offers (core/models/runs.py:708-722) distinguish 'reuses idle fleet instance' from 'provisions new cloud instance' (InstanceOfferWithAvailability.availability semantics), and whether calling get_plan from a background task is safe (session/commit behavior). -- Why: Preset selection 'matching existing fleets' is a core requirement; without the exact offer/availability semantics the plan will hallucinate a matching API or pick presets that trigger unwanted cloud provisioning. -- Where: src/dstack/_internal/server/services/runs/plan.py, src/dstack/_internal/server/services/offers.py, src/dstack/_internal/core/models/instances.py (InstanceAvailability enum), src/dstack/_internal/server/services/instances.py:723 - -### Preset storage format and source of truth are undecided: full ServiceConfiguration YAML vs partial dict, matching keys (model id normalization, GPU requirement expression), and where presets live — vendored files under src/dstack (ships in wheel), DB rows, or a git repo like the existing UITemplate system (core/models/templates.py, services/templates.py, DSTACK_SERVER_TEMPLATES_REPO). Note replicas can be a list of replica-group objects, and `model:` string coerces to OpenAIChatModel — preset schema must round-trip these. -- Why: The plan must specify preset schema, lookup order (preset vs agent fallback), and packaging; the existing templates feature also creates a naming/namespace collision to resolve deliberately. -- Where: src/dstack/_internal/core/models/templates.py, src/dstack/_internal/server/services/templates.py, src/dstack/_internal/server/routers/templates.py - -### The claude-agent-sdk Python API surface is UNVERIFIED prose: exact symbols (query() vs ClaudeSDKClient, ClaudeAgentOptions fields like allowed_tools/permission_mode/max_turns/cwd/model, custom in-process tool definition / SDK MCP server API, structured result extraction) were not pinned against the real package, nor was its footprint in the dstack server images (bundled CLI binary ~300MB, subprocess spawning inside the server container). -- Why: The agent-launch step is the least-verified part of the plan; hallucinated SDK calls or an image that can't host the bundled CLI would invalidate the whole agent path. A fallback decision (plain anthropic SDK loop) hinges on these facts. -- Where: PyPI claude-agent-sdk (pip download + inspect), https://code.claude.com/docs/en/agent-sdk/, docker/server/release/ and docker/server/Dockerfile.nebius, pyproject.toml optional-dependencies pattern (S3Storage try/except ImportError precedent) - -### Where does the Anthropic API key come from — a DSTACK_SERVER_* module constant in server/settings.py (documented in mkdocs/docs/reference/env.md) or a per-project encrypted secret via the verified secrets service (src/dstack/_internal/server/services/secrets.py:29-242, get_project_secrets_mapping/create_or_update_secret, EncryptedString)? Also: behavior when the key is absent (endpoint FAILED at submit vs feature-flag-gated router). -- Why: Determines multi-tenant cost attribution, encryption-at-rest of the key, and the failure mode for servers without a key; the plan must pick one and specify the setting name/doc update. -- Where: src/dstack/_internal/server/services/secrets.py, src/dstack/_internal/server/settings.py, src/dstack/_internal/settings.py FeatureFlags - -### What tool surface does the agent get to author/deploy the service? Options with different verified constraints: (a) in-process Python tools wrapping services.runs.apply_plan/submit_run (commits sessions multiple times — cannot run inside the pipeline worker's fetch session), (b) HTTP calls to the server's own REST API (needs a user token — UserModel token is encrypted; how to mint/decrypt one for the agent), (c) shelling out to the dstack CLI (is the CLI + config file even available/configured in the server container?). None of this was researched. -- Why: This is the central integration decision; each option changes auth, transaction handling, and idempotency, and the plan cannot be followed without picking one and naming exact functions. -- Where: src/dstack/_internal/server/services/runs/__init__.py (apply_plan/submit_run session semantics), src/dstack/_internal/server/models.py UserModel.token/EncryptedString, src/dstack/_internal/server/services/users.py (token access), docker/server/ - -### How is agent progress persisted for crash recovery? Pipeline crash-recovery re-runs the whole process() step after lock expiry, and Agent SDK sessions are replica-local (no cross-replica resume). Undecided: does EndpointModel persist an attempt counter + agent output (Text column with pydantic JSON, the established pattern), and does the agent step begin with reconciliation (get_run_by_name for the endpoint's derived run name; check RunModel.deleted at models.py:411) before launching a new agent session? -- Why: Without a persisted state machine + reconcile-first design, a replica death mid-agent-run duplicates deployments and Anthropic spend; the plan needs the exact EndpointModel columns and status enum values. -- Where: src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py (chunked state machine precedent), src/dstack/_internal/server/services/runs/__init__.py get_run_by_name - -### Out-of-band run lifecycle reconciliation: what happens when a user stops/deletes the endpoint's underlying run directly (stop_runs/delete_runs are public APIs; RunModel soft-deletes via `deleted`, models.py:411), or when the run goes FAILED/TERMINATED after the endpoint is ACTIVE? Does the endpoint pipeline keep re-checking active endpoints (min_processing_interval) and transition to FAILED / re-provision, and does endpoint deletion call stop_runs+delete_runs or leave the run? -- Why: Defines the steady-state half of the state machine and the delete flow; skipping it leaves dangling endpoints pointing at deleted runs. -- Where: src/dstack/_internal/server/services/runs/__init__.py stop_runs/delete_runs, src/dstack/_internal/server/background/pipeline_tasks/gateways.py (parent-polls-child precedent) - -### Concrete ACTIVE/FAILED criteria and deadlines: mechanism is established (JobModel.registered gated by probes; probes never fail a job), but the plan needs exact queries — how the endpoint pipeline finds the run's latest jobs and reads `registered` (RunModel.jobs relationship / latest submission filter), what provisioning deadline default applies (and its interaction with RunModel.resubmission_attempt retries), and whether 'active' means >=1 registered replica or all replicas. -- Why: Health-checking is a stated requirement; without the exact registered-job query and a timeout value the developer must re-research, and endpoints could stay 'provisioning' forever. -- Where: src/dstack/_internal/server/models.py RunModel.jobs/JobModel, src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py:1119-1130 is_job_ready, src/dstack/_internal/server/background/pipeline_tasks/runs/active.py - -### What endpoint URL is surfaced to the user as the OpenAI-compatible base URL, and how does the server know its own external base URL? ServiceSpec/model URL construction in _register_service_in_server (services/services/__init__.py:275) vs gateway domain for gateway-backed services — the exact setting (e.g. a SERVER_URL-like constant) and the returned Endpoint model fields (base_url, model name for /v1/models) were not established. -- Why: The whole point of an endpoint is a usable URL; the plan must specify what `dstack apply` prints and what the GET /endpoints API returns for both gateway and in-server-proxy cases. -- Where: src/dstack/_internal/server/services/services/__init__.py, src/dstack/_internal/server/settings.py, src/dstack/_internal/core/models/runs.py ServiceSpec/ServiceModelSpec - -### CLI/API surface decisions not pinned: does the endpoint get a server-side get_plan/apply_plan pair (gateway pattern, recommended by blueprint) or client-side plan (volume pattern); which permission dep (ProjectMember vs ProjectAdmin — precedents differ); is there a `dstack ps`-style listing or only apply/delete; and is the endpoints spec added to the plugins ApplySpec TypeVar (src/dstack/plugins/_models.py:8) or deliberately excluded? -- Why: These are one-line decisions but each one, if unspecified, forces the implementer to re-derive the pattern; the plugins TypeVar in particular breaks type-checking if forgotten. -- Where: src/dstack/_internal/server/routers/gateways.py, src/dstack/_internal/server/security/permissions.py, src/dstack/plugins/_models.py - -### What instructional context does the embedded agent receive and from where at runtime? Established: nothing agent-usable ships in the wheel (skills/, mkdocs/ excluded) and recipes need two consumption strategies (recipes.vllm.ai/models.json JSON vs SGLang MDX). Undecided: the exact vendored path under src/dstack/_internal/... for prompt/skill content, whether the agent may WebFetch at runtime (server egress policy, pytest --disable-socket means all of it must be mockable), and content update cadence/licensing attribution placement. -- Why: Agent quality depends entirely on this context; a plan that says 'load the skill' without a shippable path and a network-or-vendored decision is unimplementable. -- Where: src/dstack/_internal/server/ (pick vendoring location), pyproject.toml hatchling packaging config, skills/dstack/SKILL.md (source material to distill) - -## Risks -- Long-running agent vs replica death: heartbeater allows unbounded process(), but a crash re-runs the entire step from scratch on another replica with no agent-session resume; the endpoint state machine must be reconcile-before-act (look up existing run by derived name, check RunModel.deleted/status) and cap attempts, or crashes duplicate deployments. -- Unbounded Anthropic spend: pipeline retry semantics (row re-fetched after lock expiry, min_processing_interval re-processing) plus a flaky agent = a paid crash loop; the plan must specify max_turns/model choice, a persisted attempt counter, and a terminal FAILED state that stops retries. -- Transaction hazard: submit_run/apply_plan commit the session multiple times and take advisory locks; calling them inside the pipeline worker's token-guarded update flow breaks the lock protocol — the worker must use a fresh get_session_ctx() for run submission and treat 'run created but endpoint row update lost lock' as a reconcilable state, not corruption. -- Security boundary: the Agent SDK spawns a CLI subprocess with tool execution inside the multi-tenant dstack server container (which holds DB creds, encryption keys, cloud creds in env); tool allowlisting/permission mode and env scrubbing for the subprocess must be designed, or prompt-injected recipe content can exfiltrate server secrets. -- Event-loop starvation: agent subprocess waits and Anthropic HTTP calls run for minutes; routing them through the shared 128-thread default executor (app.py:123) or blocking the loop stalls heartbeats for ALL pipelines on the replica — endpoints pipeline needs native-async subprocess handling or a dedicated executor, and a small workers_num to bound concurrent agent runs. -- Silent history loss: submit_run deletes any finished run with the same run_name — a deterministic endpoint-derived run name erases previous attempts' logs/history on every retry; decide between attempt-suffixed names and accepting the erasure. -- Health-check false positives/negatives: RunStatus.RUNNING fires on any RUNNING replica (not model-ready), probes never fail a job (unhealthy service stays RUNNING with registered=False forever), and TGI-format models get no default probe — the endpoint needs its own registered-based check AND its own deadline to reach FAILED. -- register_service raises synchronously inside submit_run (gateway missing, FORBID_SERVICES_WITHOUT_GATEWAY, autoscaling without gateway) — agent- or preset-generated ServiceConfigurations must handle this as a submission failure path, and the agent context must know whether the target project has a gateway. -- Preset/fleet drift: a preset matched against a fleet at submit time can be invalidated before provisioning completes (fleet deleted, instances busy); matching must be advisory (offers-based) with the run's own scheduling as the source of truth, not a hard endpoint->fleet binding. -- Namespace collision: an existing 'templates' feature (UITemplate, /templates/list, DSTACK_SERVER_TEMPLATES_REPO) already occupies the config-template concept; presets must either reuse it or pick distinct naming to avoid API/docs confusion. -- Testing constraint shapes the design: pytest runs with --disable-socket and schema comes from metadata.create_all (missing migration won't fail tests), so the agent client must be injectable/mockable and the alembic migration (expand-and-contract, one table, partial pipeline-fetch index with both postgresql_where and sqlite_where) must be an explicit plan step with its own review. diff --git a/endpoint-implementation-research/dstack-prototyping-skill-audit.md b/endpoint-implementation-research/dstack-prototyping-skill-audit.md deleted file mode 100644 index 8ae6964cf6..0000000000 --- a/endpoint-implementation-research/dstack-prototyping-skill-audit.md +++ /dev/null @@ -1,115 +0,0 @@ -# dstack-prototyping Skill Audit - -Date: 2026-07-07 - -Purpose: rebuild `skills/dstack-prototyping/SKILL.md` from validated, high-value -instructions only. This file is a working audit for this branch, not user docs. - -## Skill Bar - -A sentence stays in the skill only if it passes at least one test: - -- It prevents a failure we already observed in endpoint e2e. -- It is a dstack fact verified by docs, CLI help, or source code. -- It is a fragile endpoint-agent contract the model must see during execution. -- It tells the agent how to choose between task, dev environment, service, fleet, - backend, image, resources, cache, or verification. - -A sentence is removed or moved out if it is: - -- generic project-management advice; -- a competitor/product idea that does not change the next deployment decision; -- a future optimization topic unrelated to getting a verified endpoint; -- a claim about backend behavior not supported by docs, code, run evidence, or - harness-provided facts; -- a restatement of `/dstack` CLI syntax unless the endpoint harness depends on - the distinction. - -## Validated Facts - -| Fact | Source | Decision | -| --- | --- | --- | -| dstack has VM-based and container-based backends. | `mkdocs/docs/concepts/backends.md` | Keep. Critical for experiment strategy. | -| VM-based backends give dstack native control over provisioning. | `mkdocs/docs/concepts/backends.md` | Keep, but avoid saying every VM backend is always better. | -| Container-based backends delegate provisioning to the provider or Kubernetes. | `mkdocs/docs/concepts/backends.md` | Keep, but do not overgeneralize to no cache/no SSH without evidence. | -| SSH fleets exist without backend configuration and use user/admin-managed hosts. | `mkdocs/docs/concepts/backends.md`, `mkdocs/docs/concepts/fleets.md` | Keep. Strong signal for interactive prototyping. | -| Fleets must exist before runs. | `mkdocs/docs/concepts/fleets.md`, task/dev/service docs | Keep. Endpoint agent must not create fleets. | -| Backend fleets with `nodes` starting at `0` create only a template; instances are provisioned by runs. | `mkdocs/docs/concepts/fleets.md` | Keep for reasoning about cold starts. | -| Pre-provisioning is supported only for VM-based backends. | `mkdocs/docs/concepts/fleets.md` | Keep. Important distinction from container backends. | -| Fleet `idle_duration` keeps idle backend-fleet instances around for reuse. | `mkdocs/docs/concepts/fleets.md`, snippets | Keep. Explains warm-instance strategy. | -| `dstack offer` ignores fleet configs by default. `--fleet` restricts offers to selected fleets. | `mkdocs/docs/reference/cli/dstack/offer.md` | Keep. Prevents global-offer mistakes. | -| `dstack offer --group-by gpu,backend` is documented and `gpu` must be included. | `mkdocs/docs/reference/cli/dstack/offer.md`, CLI help | Keep. Useful for multi-backend fleets. | -| Offer output is hardware/capacity/price, not proof of backend implementation class. | Inference from offer docs + observed failure | Keep as a guardrail. Label as interpretation, not doc quote. | -| Dev environments are accessible via IDE or SSH. | `mkdocs/docs/concepts/dev-environments.md` | Keep. Useful for interactive probe choice. | -| Tasks run arbitrary commands and can be single-node or distributed. | `mkdocs/docs/concepts/tasks.md` | Keep only the part needed for probes. | -| Services are final endpoint artifacts and can expose OpenAI-compatible model URLs when `model` is set. | `mkdocs/docs/concepts/services.md`, `/dstack` skill | Keep. Final proof depends on it. | -| Resource ranges are supported for CPU/memory/GPU/disk. | task/dev docs, offer reference, CLI help | Keep briefly. | -| GPU spec can include vendor/model/memory/count. | offer reference, dev docs, CLI help | Keep for vendor-aware recipes. | -| Instance volumes can be optional; optional volumes run even if the selected backend cannot mount them. | `mkdocs/docs/concepts/volumes.md` | Keep. Useful for cache mounts without over-constraining placement. | -| Instance volumes are not supported on RunPod and Vast.ai in docs. | `mkdocs/docs/concepts/volumes.md` | Keep carefully; do not extrapolate to all persistence. | -| Local code exposes backend feature lists for create-instance, instance-volumes, privileged, multinode. | `src/dstack/_internal/core/backends/features.py` | Keep if harness injects them. | -| `ComputeWithCreateInstanceSupport` is documented in code as fleet instance creation without running a job; typically VMs implement it and containers do not. | `src/dstack/_internal/core/backends/base/compute.py` | Keep as implementation signal, not public UX. | -| JarvisLabs supports create-instance, privileged, and instance-volumes in this build. | `src/dstack/_internal/core/backends/jarvislabs/compute.py` | Keep in harness hints, not hardcoded in the generic skill. | -| RunPod supports group provisioning, multinode, and network volumes but not create-instance or instance-volumes in this build. | `src/dstack/_internal/core/backends/runpod/compute.py`, features list | Keep in harness hints, not hardcoded in the generic skill. | -| `python` and `image` are mutually exclusive in run configs. | `/dstack` skill | Do not duplicate unless endpoint failures show agents violate it. | - -## Observed Endpoint-Agent Failures To Prevent - -| Failure | Skill/prompt rule | -| --- | --- | -| Agent called JarvisLabs container-style because RunPod rows appeared first in offers. | Classify allowed backends from docs/code/harness facts before relying on offers; offers are not backend-class proof. | -| Agent chose RunPod by omission because service/task YAML did not pin `backends`. | If YAML omits `backends`, admit scheduler may choose any matching backend allowed by constraints. | -| Agent packed a whole probe into task `commands` instead of making an interactive target. | Probe task should stay alive when attach/SSH is available; run checks through attach/SSH. | -| Agent promoted based on shallow host evidence. | `nvidia-smi` alone is not recipe proof. Probe must test the intended image/framework/model/server slice. | -| Agent used `latest` image and hit driver/runtime mismatch. | Final service image should be pinned or justified by source/probe. | -| Agent resubmitted services and paid cold-start cost repeatedly. | Prefer task/dev probes and reusable/inspectable backends when they can reduce total iteration. | -| Endpoint logs had vague categories and little reasoning. | Progress must explain decisions and evidence in natural language. | -| Service reached healthy before endpoint became running. | Final service must still be verified by agent and reported; endpoint success is not service status alone. | - -## Keep In Skill - -- Load `/dstack` first and do not duplicate CLI syntax. -- Define success as verified final service model request. -- Require the agent to decide the proof request, model facts, framework, resource envelope, - constraints, and first experiment before paid runs, without forcing a markdown note. -- Require existing fleets; endpoint agents do not create/edit fleets. -- Use fleet-filtered offers, not global offers, when endpoint capacity is fleet-bounded. -- Compare multi-backend fleets with the intended resource envelope. -- Classify backend choice from docs/code/harness/run evidence, not truncated offers. -- Prefer inspectable/reusable placements when viable, but allow container paths with a reason. -- Derive minimum resources separately from selected offers and validation hardware. -- Use vendor-aware GPU resources after the recipe or placement is vendor-specific. -- Verify driver/runtime when it affects image/framework choice. -- Prefer task/dev probe before final service for new recipes on unverified paths. -- Make probes interactive when attach/SSH is available. -- Probe the intended serving stack, not just GPU visibility. -- Pin final images or justify them from source/probe. -- Use optional instance cache mounts when useful and supported; do not assume persistence. -- Use run JSON/events/logs/attached shell for different evidence types. -- Classify failures before changing YAML. -- Promote only a clean service and verify the model URL with a real request. -- Keep endpoint progress human-readable and structured endpoint files parseable. - -## Remove From Skill - -- Long competitor-link explanations. Keep references in research/test-plan docs; the skill should not push v1 agents toward benchmarking or kernel work. -- Generic "do not chase local optimum" phrasing. Replace with concrete decision gates. -- Full lists of every possible failure unless they guide the immediate next edit. -- Detailed final report schema duplication that belongs in the endpoint prompt. -- Broad claims such as "VM is better" without constraints. -- Any backend-specific fact that can be injected by the harness for the actual project. - -## Rebuild Shape - -The rebuilt `SKILL.md` should be short enough for the agent to actually follow: - -1. Scope and success gate. -2. Pre-run decisions. -3. Fleet/backend choice. -4. Hardware/resources. -5. Experiment selection. -6. Probe quality. -7. Images/cache. -8. Evidence/failure routing. -9. Promotion/final verification. -10. Endpoint progress and structured files. diff --git a/endpoint-implementation-research/dstack-prototyping-skill-outline.md b/endpoint-implementation-research/dstack-prototyping-skill-outline.md deleted file mode 100644 index ab4a7bfa6d..0000000000 --- a/endpoint-implementation-research/dstack-prototyping-skill-outline.md +++ /dev/null @@ -1,11 +0,0 @@ -# dstack-prototyping Skill Scorecard - -Score every bullet from 0 to 10. Each bullet should stand alone and cover one complete idea. - -- Purpose: use this skill to find a working dstack service recipe for a requested model by testing the recipe in a task on real hardware before submitting the final service. -- Dependency: this skill must be used together with the `dstack` skill; `dstack` provides CLI/YAML syntax, while this skill explains how to prototype model serving with dstack runs. -- Fleet/backend choice: prefer allowed fleets/backends that can leave an idle instance available after a run so repeated task/service attempts can reuse the provisioned machine and local caches, especially model weights; judge this from fleet config, current fleet state, backend docs, and observed run behavior, not from price alone. -- Interactive access: attach, SSH, or Kubernetes exec is useful for inspecting and adjusting a live task, but it is separate from idle instance reuse; prefer both when available. -- Serving sources: check serving-framework sources early enough to choose image, command, flags, resources, cache paths, request format, and expected model behavior; for vLLM and SGLang, treat the listed recipe/docs/cookbook sources as credible starting points. -- Task workflow: before submitting a service, start a long-lived task with `sleep infinity` or equivalent, attach/SSH when available, and test image, installs, model download/cache, serving command, port, launch flags, local model request, and expected model behavior; if image, hardware, or install path changes, submit another task. -- Service workflow: submit the service after the task verifies the recipe, then repeat the model request through the dstack service URL; if the fix requires changing image/install/download/command/resources/cache/model behavior, return to a task, otherwise fix and resubmit the service config. diff --git a/endpoint-implementation-research/examples-llm-deployments.md b/endpoint-implementation-research/examples-llm-deployments.md deleted file mode 100644 index 46f4fceb7e..0000000000 --- a/endpoint-implementation-research/examples-llm-deployments.md +++ /dev/null @@ -1,129 +0,0 @@ -# examples-llm-deployments - -## Summary -The repo's LLM serving examples now live primarily as YAML embedded in mkdocs docs pages (mkdocs/docs/examples/inference/{vllm,sglang,trtllm,nim,dynamo}.md); the examples/ directory itself contains only ONE serving .dstack.yml (examples/models/gpt-oss/amd/120b.dstack.yml). All examples are plain `type: service` configs using image + commands + port + model + env + volumes + resources.gpu; none use spot_policy. A preset-like mechanism ALREADY EXISTS: a "UI templates" system (core/models/templates.py UITemplate, server/services/templates.py, routers/templates.py) that pulls template YAMLs from a git repo (DSTACK_SERVER_TEMPLATES_REPO or per-project templates_repo) with an embedded `configuration: Dict[str, Any]` — a strong precedent/reuse candidate for endpoint presets. No "preset" or "recipe" concept exists anywhere in src/. - -## Key files -- mkdocs/docs/examples/inference/vllm.md — — vLLM service YAML (NVIDIA + AMD tabs), Qwen/Qwen3.6-27B on vllm/vllm-openai:v0.19.1 -- mkdocs/docs/examples/inference/sglang.md — — SGLang YAML incl. replica-groups PD-disaggregation example with `replicas: [{count, commands, resources, router|scaling}]` -- mkdocs/docs/examples/inference/nim.md — — NIM YAML: registry_auth with NGC_API_KEY, no commands (image entrypoint), gpu: H100:80GB:8 -- mkdocs/docs/examples/inference/trtllm.md — — TensorRT-LLM YAML: trtllm-serve, gpu: H100:8 -- mkdocs/docs/examples/inference/dynamo.md — — Dynamo PD-disaggregation with replica groups + scaling (metric: rps) -- examples/models/gpt-oss/amd/120b.dstack.yml — — Only real serving .dstack.yml file in examples/; vLLM on MI300X:8, env includes HF_TOKEN -- src/dstack/_internal/core/models/templates.py — UITemplate, AnyUITemplateParameter — Existing template model: type: template, name, title, description, parameters[], configuration: Dict[str,Any] -- src/dstack/_internal/server/services/templates.py — list_templates(project), _fetch_templates_repo, TEMPLATES_DIR_NAME='.dstack/templates' — Clones/pulls a git repo into SERVER_DATA_DIR_PATH/templates-repos/, parses YAML templates, TTLCache 180s -- src/dstack/_internal/server/routers/templates.py — POST /api/project/{project_name}/templates/list — UI-facing list endpoint, ProjectMember permission -- src/dstack/_internal/server/settings.py — SERVER_TEMPLATES_REPO (line 145) — env var DSTACK_SERVER_TEMPLATES_REPO; per-project override via ProjectModel.templates_repo -- mkdocs.yml — — nav lines 329-334 index Inference examples (SGLang, Dynamo, vLLM, NIM, TensorRT-LLM); many redirects show examples were moved from examples/ into docs - -## Details -## 1. Where LLM serving examples live - -The `examples/` directory was heavily trimmed. Current top-level dirs: `distributed-training, llms, misc, models, plugins, server-deployment, single-node-training`. The ONLY serving `.dstack.yml` on disk is `examples/models/gpt-oss/amd/120b.dstack.yml` (vLLM/ROCm). `examples/llms/deepseek/trl/*` are training tasks, not services. There are no per-example README.md files for inference in examples/ (READMEs exist only for misc/, plugins/, qlora, cloudformation). - -The canonical serving examples are YAML blocks embedded in markdown under `mkdocs/docs/examples/inference/`: -- `vllm.md` — vLLM (NVIDIA H100:4 + AMD MI300X:4) -- `sglang.md` — SGLang (incl. replica-groups PD disaggregation) -- `trtllm.md` — TensorRT-LLM -- `nim.md` — NVIDIA NIM -- `dynamo.md` — Dynamo (PD disaggregation, autoscaling) -Also `mkdocs/docs/examples/models/{deepseek-v4.md, qwen36.md}` and `mkdocs/docs/examples/accelerators/*.md`. - -Indexing: `mkdocs.yml` nav lines 316-341 (Examples > Training/Clusters/Inference/Models/Accelerators) plus a hand-written card grid in `mkdocs/docs/examples.md`. No metadata files. There is no TGI service example anymore (TGI appears only in the proxy model-client `src/dstack/_internal/proxy/lib/services/model_proxy/clients/tgi.py`, `format: tgi` in the model spec). - -## 2. Representative full YAMLs (verbatim from repo) - -vLLM (mkdocs/docs/examples/inference/vllm.md, NVIDIA tab): -```yaml -type: service -name: qwen36 -image: vllm/vllm-openai:v0.19.1 -commands: - - | - vllm serve Qwen/Qwen3.6-27B \ - --host 0.0.0.0 \ - --port 8000 \ - --tensor-parallel-size $DSTACK_GPUS_NUM \ - --max-model-len 262144 \ - --reasoning-parser qwen3 -port: 8000 -model: Qwen/Qwen3.6-27B -volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true -resources: - shm_size: 16GB - gpu: H100:4 -``` -AMD variant differs only in image (`vllm/vllm-openai-rocm:v0.19.1`) and resources (`cpu: 52..`, `memory: 896GB..`, `disk: 450GB..`, `gpu: MI300X:4`). - -SGLang (sglang.md, NVIDIA): same shape with `image: lmsysorg/sglang:v0.5.10.post1`, `commands: sglang serve --model-path Qwen/Qwen3.6-27B --tp $DSTACK_GPUS_NUM ...`, `port: 30000`, `model: Qwen/Qwen3.6-27B`, same cache volume, `resources: {shm_size: 16GB, gpu: H100:4}`. - -NIM (nim.md) — no commands, uses registry_auth + env passthrough: -```yaml -type: service -name: nemotron120 -image: nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:1.8.0 -env: - - NGC_API_KEY -registry_auth: - username: $oauthtoken - password: ${{ env.NGC_API_KEY }} -port: 8000 -model: nvidia/nemotron-3-super-120b-a12b -volumes: - - instance_path: /root/.cache/nim - path: /opt/nim/.cache - optional: true -resources: - cpu: x86:96.. - memory: 512GB.. - shm_size: 16GB - disk: 500GB.. - gpu: H100:80GB:8 -``` - -Real on-disk example `examples/models/gpt-oss/amd/120b.dstack.yml` uses `env: [HF_TOKEN, MODEL=openai/gpt-oss-120b, VLLM_ROCM_USE_AITER=1, ...]` (HF_TOKEN as passthrough), short-form volume `- /root/.cache/huggingface:/root/.cache/huggingface`, `gpu: MI300X:8`. - -Fields observed across all serving examples: `type: service`, `name`, `image`, `commands` (multiline `|` blocks), `port`, `model` (plain HF model-id string), `env` (list; passthrough names like `HF_TOKEN`/`NGC_API_KEY` and `KEY=value` pairs), `volumes` (instance_path/path/optional or short form), `resources` (`gpu: :`, `gpu: H100:80GB:8` with mem, ranges `cpu: 96..`, `memory: 512GB..`, `disk:`, `shm_size:`), `registry_auth`. Advanced: `replicas` as a LIST of replica groups `{count: 1 or 1..4, commands, resources, scaling: {metric: rps, target: 3}, router: {type: sglang}}` (sglang.md line ~155, dynamo.md line 27). `spot_policy` appears in NO example (only in concepts/services.md docs, values `spot|on-demand|auto`). Simple scalar `replicas: N` + `scaling` also documented in sglang/dynamo autoscaling notes. - -## 3. Existing preset/recipe/template machinery - -- "preset"/"recipe": do not exist as concepts anywhere in src/ (grep hits for those words are incidental — e.g. nebius "preset" is a cloud API field, runpod/cloudrift "template" is provider API terminology). -- "template" DOES exist as a real feature: **UI Templates** (added Mar 2026, migration `03_06_1200_a13f5b55af01_add_projectmodel_templates_repo.py`). - - Model: `UITemplate` in `src/dstack/_internal/core/models/templates.py:59` — `{type: "template", name, title, description, parameters: List[AnyUITemplateParameter], configuration: Dict[str, Any]}` where `configuration` is a raw dstack run configuration dict. Parameter types: name, ide, resources, python_or_docker, repo, working_dir, env (with title/name/value). - - Service: `src/dstack/_internal/server/services/templates.py` — `async list_templates(project) -> List[UITemplate]`; clones a git repo (`project.templates_repo` or `settings.SERVER_TEMPLATES_REPO` = env `DSTACK_SERVER_TEMPLATES_REPO`, settings.py:145) into `SERVER_DATA_DIR_PATH/templates-repos/`, reads YAML files from `.dstack/templates` dir in the repo, TTL-cached 180s, run in thread via `run_async`. - - Router: `src/dstack/_internal/server/routers/templates.py` — `POST /api/project/{project_name}/templates/list`, `ProjectMember()` permission, registered in `server/app.py`. - - DB: `ProjectModel.templates_repo` column in `server/models.py`. - - These templates are consumed by the UI only; there is no server-side "instantiate template into a run" logic in src/. - -## 4. Preset sketch grounded in real syntax - -A preset could be a stored service config keyed by model + accelerator, e.g.: -```yaml -# preset for Qwen/Qwen3.6-27B on NVIDIA -model: Qwen/Qwen3.6-27B -service: # verbatim dstack service configuration (like UITemplate.configuration) - type: service - image: vllm/vllm-openai:v0.19.1 - commands: - - | - vllm serve Qwen/Qwen3.6-27B --host 0.0.0.0 --port 8000 \ - --tensor-parallel-size $DSTACK_GPUS_NUM --max-model-len 262144 - port: 8000 - model: Qwen/Qwen3.6-27B - env: - - HF_TOKEN # passthrough, merged with endpoint's env - volumes: - - instance_path: /root/.cache - path: /root/.cache - optional: true - resources: - shm_size: 16GB - gpu: H100:4 # matching key vs fleet GPUs -``` -Every field above is verified real service syntax. The existing UITemplate pattern (raw `configuration: Dict[str, Any]` validated later against the run-configuration parser) is the natural precedent for storing the embedded service config. - -## Gotchas -1) Do NOT assume examples/ contains vLLM/SGLang/TGI .dstack.yml files — it doesn't; the serving YAMLs live only inside mkdocs markdown, so a preset library cannot be built by globbing examples/. 2) There is no TGI serving example anymore (TGI survives only as a proxy model `format`). 3) `replicas` in current examples is a LIST of replica-group objects (count/commands/resources/scaling/router), not just an int — a preset schema must allow both. 4) `model:` in service YAML is a plain string (HF model id); older `model: {type/name/format}` object form was not seen in these examples. 5) No example uses spot_policy; don't copy it into presets as if idiomatic. 6) An existing "templates" feature (UITemplate + git-repo fetch + /templates/list API) already occupies the template namespace — the endpoint plan should either reuse it or deliberately name presets differently to avoid collision. 7) GPU spec strings like `H100:80GB:8` and range syntax `cpu: 96..` / `memory: 512GB..` are valid and used; presets should preserve exact string forms. diff --git a/endpoint-implementation-research/external-recipes.md b/endpoint-implementation-research/external-recipes.md deleted file mode 100644 index 42592b3e78..0000000000 --- a/endpoint-implementation-research/external-recipes.md +++ /dev/null @@ -1,85 +0,0 @@ -# external-recipes - -## Summary -Both projects exist and are Apache-2.0 licensed, so an embedded dstack agent can fetch, vendor, or redistribute their content with attribution. The SGLang Cookbook's canonical home is now docs.sglang.io/cookbook (source: sgl-project/sglang repo under docs_new/cookbook, MDX files); the old sgl-cookbook GitHub repo was archived read-only on 2026-06-11 and cookbook.sglang.io is legacy. vLLM recipes live at github.com/vllm-project/recipes and render at recipes.vllm.ai, which exposes a purpose-built machine-readable JSON API (/models.json index + per-model JSON with exact vllm serve commands, docker commands, and per-GPU hardware configs) — ideal for runtime WebFetch by an agent. SGLang has an llms.txt full-doc index at docs.sglang.io/llms.txt and serves page source by appending .md, but pages are MDX mixing static serve commands/hardware tables with JSX config-generator components, so parsing is messier; for SGLang, vendoring/sparse-cloning docs_new/cookbook or the archived repo's markdown is more robust. - -## Key files -- https://recipes.vllm.ai/models.json — — EXTERNAL URL (no local files in this research). Verified machine-readable index of ~500 model entries; fields per entry: hf_id, title, provider, url, json (path to per-model JSON), optional derived_from. -- https://recipes.vllm.ai/Qwen/Qwen3-32B.json — — EXTERNAL URL. Verified per-model JSON: meta (title/description/hardware verification), model (parameter_count, context_length, min vLLM version), recommended_command (exact vllm serve string per hardware), docker_command (vllm/vllm-openai:latest), by_hardware map (H200/H100/B200/TPU), variants (BF16/FP8/AWQ), guide section. -- https://github.com/vllm-project/recipes — — EXTERNAL URL. Canonical vLLM recipes repo, Apache-2.0 (LICENSE verified raw). New format: structured YAML at models//.yaml mirroring HuggingFace paths; legacy markdown per org (Qwen/Qwen3.5.md, GLM/GLM5.md, Google/Gemma4.md) still present but being migrated. JSON API rebuilt via scripts/build-recipes-api.mjs. -- https://raw.githubusercontent.com/vllm-project/recipes/main/models/deepseek-ai/DeepSeek-V4-Pro.yaml — — EXTERNAL URL. Verified concrete YAML recipe: model_id deepseek-ai/DeepSeek-V4-Pro (1.6T MoE/49B active, 1,048,576 ctx), verified hardware matrix (H200/B200/B300/GB200/GB300/MI355X; MI300X unsupported), base args --trust-remote-code --kv-cache-dtype fp8 --block-size 256, variants (default FP4+FP8, nvfp4, dspark; 960 GB VRAM min), parallelism strategies (TP/TEP/DEP/multi-node/PD), per-hardware overrides, AMD docker image vllm/vllm-openai-rocm:nightly. -- https://docs.sglang.io/llms.txt — — EXTERNAL URL. Verified: full documentation index including every cookbook recipe; recipe URL pattern https://docs.sglang.io/cookbook/autoregressive//.md (appending .md returns raw MDX source). -- https://docs.sglang.io/cookbook/autoregressive/intro — — EXTERNAL URL. Canonical SGLang Cookbook location. 23 vendor families listed (Qwen/Qwen3.6, DeepSeek/DeepSeek-V4, Llama/Llama3.3-70B, GLM/GLM-5.2, Moonshotai/Kimi-K2.7-Code, OpenAI/GPT-OSS, MiniMax, NVIDIA, Mistral, etc.); also a diffusion section and benchmark pages. -- https://github.com/sgl-project/sglang/tree/main/docs_new — — EXTERNAL URL. Current cookbook SOURCE after migration: cookbook/autoregressive//.mdx (Mintlify MDX + docs.json nav). sglang repo LICENSE verified Apache-2.0 ('Copyright 2023-2024 SGLang Team'). Sparse-checkout of docs_new/cookbook is the vendoring target. -- https://github.com/sgl-project/sgl-cookbook — — EXTERNAL URL. Old cookbook repo — ARCHIVED read-only 2026-06-11, Apache-2.0. Still useful frozen snapshot: plain markdown at docs/autoregressive//.md plus structured YAML configs at data/models/src|generated/v/.yaml (e.g. deepseek.yaml, qwen.yaml) feeding interactive config generators. -- https://raw.githubusercontent.com/sgl-project/sgl-cookbook/main/docs/autoregressive/DeepSeek/DeepSeek-V3_2.md — — EXTERNAL URL. Verified concrete SGLang recipe example: exact 'sglang serve --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --host 0.0.0.0 --port 30000' commands (plus reasoning-parser/tool-call variants), hardware guidance (8x B200 TP=8; 16x H800 TP=16 across 2 nodes; H200/B200/MI355X supported), sections: Introduction, Installation, Deployment, Invocation, Benchmark. - -## Details -## 1. SGLang Cookbook - -**Canonical source (verified):** https://docs.sglang.io/cookbook/intro — the cookbook migrated into the main SGLang docs. Source files live in the main repo at `github.com/sgl-project/sglang/tree/main/docs_new/cookbook` as **Mintlify MDX** (e.g. `cookbook/autoregressive/Qwen/Qwen3.5.mdx`). The standalone repo `github.com/sgl-project/sgl-cookbook` (previously rendered at cookbook.sglang.io) was **archived read-only on 2026-06-11**; its README points contributors to `sglang/docs_new`. I did not fetch cookbook.sglang.io itself, so whether it still serves or redirects is unverified. - -**Structure:** `cookbook/autoregressive//` and `cookbook/diffusion/...`, plus benchmark pages. 23 vendor families verified in the autoregressive index, including `DeepSeek/DeepSeek-V4`, `Qwen/Qwen3.6`, `Llama/Llama3.3-70B`, `GLM/GLM-5.2`, `Moonshotai/Kimi-K2.7-Code`, `OpenAI/GPT-OSS`, `MiniMax/MiniMax-M3`, `NVIDIA/Nemotron3-Ultra`, `Mistral/Ministral-3`. The llms.txt index also lists older recipes retained after migration (DeepSeek-R1/V3/V3.1/V3.2, Qwen2.5-VL/Qwen3/Qwen3-Coder/Qwen3.5, Kimi-K2/K2.5/K2.6/Kimi-Linear). - -**Per-recipe content (verified on two recipes):** -- Exact serve commands. From the DeepSeek-V3.2 recipe (fetched raw from the archived repo): - ``` - sglang serve \ - --model deepseek-ai/DeepSeek-V3.2-Exp \ - --tool-call-parser deepseekv31 \ - --reasoning-parser deepseek-v3 \ - --chat-template ./examples/chat_template/tool_chat_template_deepseekv32.jinja \ - --tp 8 --host 0.0.0.0 --port 30000 - ``` - with hardware guidance: 8x B200 (TP=8) or 16x H800 (TP=16, 2 nodes); supported platforms H200/B200/MI355X. -- Docker image tags. From the current Qwen3.5 MDX: `lmsysorg/sglang:latest` (NVIDIA), `lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi30x-20260604` (MI300X/MI325X), `...mi35x-...` (MI355X); AMD commands use env prefixes like `SGLANG_USE_AITER=1 ... python3 -m sglang.launch_server`. -- GPU sizing tables: e.g. Qwen3.5-397B-A17B — "H100 (80GB) requires tp=16 (2 nodes)" for BF16 vs "tp=8" for FP8; a table maps each GPU to memory + TP across precisions. -- Benchmarks (TTFT/TPOT/tokens-per-sec at multiple concurrencies) and tuning sections. - -**Format caveat (verified):** current pages are MDX — ~90% static text (commands, tables, docker pulls are literal) plus an embedded interactive JSX component (e.g. ``, `Playground` with `dockerImages` config objects). Some pages (DeepSeek-V4) are more JSX-heavy, with commands generated dynamically from embedded config objects rather than written as static shell blocks. The archived sgl-cookbook additionally has structured YAML per model at `data/models/src/v/.yaml` (frozen as of June 2026) — the closest thing SGLang has to machine-readable recipes, but no longer maintained there; whether equivalent YAML lives in `docs_new` is **unverified**. - -## 2. vLLM recipes - -**Canonical source (verified):** https://github.com/vllm-project/recipes, rendered at https://recipes.vllm.ai (144 recipes across 40+ providers, per the site). - -**Structure (verified):** two generations coexist: -- Legacy markdown: top-level org dirs (`Qwen/Qwen3.5.md`, `GLM/GLM5.md`, `Google/Gemma4.md`, ...) — being migrated. -- New structured YAML: `models//.yaml`, path mirrors HuggingFace so `recipes.vllm.ai//` matches `huggingface.co//`. Validated/compiled to a JSON API via `node scripts/build-recipes-api.mjs`; CONTRIBUTING says static JSON is exported under `public/` explicitly "for agents/tools to consume" (that quote from a search snippet; the working endpoints below corroborate it). - -**Concrete example 1 (YAML, verified raw fetch):** `models/deepseek-ai/DeepSeek-V4-Pro.yaml` — model_id `deepseek-ai/DeepSeek-V4-Pro` (1.6T total / 49B active MoE, 1,048,576-token context); verified-hardware matrix H200/B200/B300/GB200/GB300/MI355X (MI300X/MI325X marked unsupported); base args `--trust-remote-code --kv-cache-dtype fp8 --block-size 256`; variants default(FP4+FP8)/nvfp4/dspark, each min ~960 GB VRAM; parallelism strategies (default `single_node_tep`, plus TP/DEP/multi-node/PD-cluster); per-hardware overrides (Hopper: max-model-len capped at 800k; Blackwell: `--moe-backend deep_gemm_mega_moe`; AMD: `--distributed-executor-backend mp`, docker `vllm/vllm-openai-rocm:nightly`). - -**Concrete example 2 (rendered JSON, verified):** `https://recipes.vllm.ai/Qwen/Qwen3-32B.json` — includes `"recommended_command": {"hardware": "h200", "command": "vllm serve Qwen/Qwen3-32B \\\n --tensor-parallel-size 1 \\\n --reasoning-parser qwen3"}`, a full `docker_command` using image `vllm/vllm-openai:latest`, `by_hardware` configs (H200/H100/B200/TPU), quantization `variants` (BF16/FP8/AWQ), model metadata (32B params, 40960 ctx, min vLLM version), and a prose `guide`. - -## 3. Licenses (both verified from LICENSE files) - -- `vllm-project/recipes`: **Apache License 2.0** (raw LICENSE fetched). -- `sgl-project/sglang` (hosts docs_new/cookbook): **Apache License 2.0**, "Copyright 2023-2024 SGLang Team" (raw LICENSE fetched). The archived `sgl-cookbook` repo is also Apache-2.0 (per its repo page). - -Implication: runtime fetching, caching, vendoring, and redistribution of recipe content by a dstack-embedded agent are all permitted; if vendoring, retain the Apache-2.0 license text and attribution notices. (Note: the licenses cover the recipe text/config, not the model weights they describe.) - -## 4. Runtime WebFetch vs clone/vendor - -**vLLM — fetch at runtime, it's designed for it:** -- Index: `GET https://recipes.vllm.ai/models.json` (verified; ~500 entries incl. quantized variants; fields hf_id/title/provider/url/json/derived_from). -- Per model: `GET https://recipes.vllm.ai//.json` (verified) — already contains the exact serve command per hardware; no MD parsing needed. -- Fallback/stable raw: `https://raw.githubusercontent.com/vllm-project/recipes/main/models//.yaml` (verified working). -- Churn risk: legacy `*/.md` paths are mid-migration — don't build against those. - -**SGLang — fetchable but messier; vendoring is safer:** -- Discoverable at runtime via `https://docs.sglang.io/llms.txt` (verified index) and per-page raw source by appending `.md` (verified: returns MDX source, not clean markdown). -- Risk: content is MDX with JSX components (some recipes like DeepSeek-V4 keep docker images/args inside JSX config objects, not shell blocks), and the cookbook already changed homes once within a month (cookbook.sglang.io → docs.sglang.io), so URL churn risk is real. -- Vendoring option: sparse-checkout of `sglang/docs_new/cookbook` (current, MDX) or the frozen archived `sgl-cookbook` (plain md + structured `data/models/src/*.yaml`, but stale post-2026-06-11). - -## 5. Machine-readable indexes - -- **vLLM:** `recipes.vllm.ai/models.json` + per-model `//.json` (both verified). No llms.txt — `https://recipes.vllm.ai/llms.txt` returns **404** (verified). -- **SGLang:** `https://docs.sglang.io/llms.txt` (verified; full doc/cookbook index with .md URLs — Mintlify-style). No JSON recipe API found for SGLang; the archived repo's `data/models/src|generated/v*/{model}.yaml` config-generator data is the nearest equivalent (frozen). Whether docs.sglang.io exposes llms-full.txt or an equivalent JSON is **unverified** (not checked). - -## Unverified items (explicit) -- Current behavior of cookbook.sglang.io (redirect vs stale mirror) — not fetched. -- Whether every docs.sglang.io page reliably serves source via the `.md` suffix (verified only for DeepSeek-V4.md). -- The `public/` static-JSON claim in vLLM CONTRIBUTING.md was seen only in a search snippet (endpoints themselves verified live). -- Post-knowledge-cutoff model names (DeepSeek-V4, Qwen3.5/3.6, GLM-5.2, Kimi-K2.7-Code, etc.) are reported exactly as returned by the fetched pages. - -## Gotchas -1) The "SGLang Cookbook" GitHub repo (sgl-project/sgl-cookbook) is ARCHIVED (read-only since 2026-06-11) — do not plan around it as the live source; the live source is sgl-project/sglang:docs_new/cookbook and the live site is docs.sglang.io/cookbook (NOT cookbook.sglang.io). 2) SGLang recipe pages are MDX, not plain markdown: appending .md to a docs.sglang.io URL returns raw MDX where some recipes (e.g. DeepSeek-V4) embed docker images and serve-arg matrices inside JSX Playground/ConfigGenerator components rather than static shell blocks — a naive markdown-reading agent will miss or misparse those; older/most recipes (e.g. Qwen3.5) are ~90% static and fine. 3) SGLang cookbook URLs churned once already (repo + domain moved in June 2026), so hardcoded deep links are fragile; resolve via docs.sglang.io/llms.txt at runtime. 4) vLLM recipes repo is mid-migration from per-org markdown to models//.yaml — build against recipes.vllm.ai/models.json + per-model JSON (or raw YAML on GitHub), never the legacy .md paths. 5) recipes.vllm.ai has NO llms.txt (404) — its discovery entry point is /models.json; conversely SGLang has llms.txt but NO JSON API. An agent needs two different consumption strategies. 6) Apache-2.0 on both covers the recipe text only, not the model weights the recipes deploy; vendored copies must retain LICENSE/attribution. 7) vLLM's models.json includes ~500 entries but many are quantized variants linked via derived_from — dedupe on that field when presenting model choices. diff --git a/endpoint-implementation-research/resource-lifecycle-blueprint.md b/endpoint-implementation-research/resource-lifecycle-blueprint.md deleted file mode 100644 index b5c9b5dde6..0000000000 --- a/endpoint-implementation-research/resource-lifecycle-blueprint.md +++ /dev/null @@ -1,201 +0,0 @@ -# resource-lifecycle-blueprint - -## Summary -Gateways and volumes are the two existing non-run resources with a full provisioning lifecycle, and they share one architecture: a SQLAlchemy model with PipelineModelMixin (lock_expires_at/lock_token/lock_owner) + status/status_message/last_processed_at/to_be_deleted columns; a status enum in core/models with SUBMITTED -> (PROVISIONING) -> RUNNING/ACTIVE | FAILED; a service module with create_*/delete_*/list_* functions raising ServerClientError subclasses; a Pipeline subclass (Fetcher/Worker/Heartbeater) in server/background/pipeline_tasks registered in PipelineManager; a FastAPI router with ProjectAdmin (gateways) or ProjectMember (volumes) permission deps; a BaseApplyConfigurator subclass wired into apply_configurators_mapping for `dstack apply`/`dstack delete`; and an APIClientGroup on APIClient. Deletion is asynchronous in both cases: API handlers only set to_be_deleted=True, and the pipeline worker performs backend cleanup and then hard-deletes the row (gateway) or soft-deletes with deleted=True/deleted_at (volume). All processing is multi-replica safe via SELECT ... FOR UPDATE SKIP LOCKED + lock_token/lock_expires_at leases with a heartbeat task, and update-after-processing is written via UPDATE ... WHERE id=... AND lock_token=... so lost locks are detected. I verified every symbol below by reading the code. - -## Key files -- src/dstack/_internal/server/models.py — PipelineModelMixin (L204), GatewayModel (L600), GatewayComputeModel (L656), VolumeModel (L951), VolumeAttachmentModel (L1001) — DB models. UUID pk (UUIDType(binary=False), default=uuid.uuid4), project FK with ondelete CASCADE, status stored as EnumAsString(StatusEnum, 100), Text status_message, NaiveDateTime last_processed_at, Boolean to_be_deleted (server_default=false()), JSON blobs stored as Text (configuration, volume_provisioning_data). Partial index ix_
_pipeline_fetch_q on last_processed_at.asc() filtered by deleted==false. -- src/dstack/_internal/core/models/gateways.py — GatewayStatus (L17), GatewayReplicaStatus (L24), GatewayConfiguration (L59, type: Literal["gateway"]), GatewaySpec (L125), Gateway (L141), GatewayPlan (L176), ApplyGatewayPlanInput (L185) — Core (client-visible) pydantic models. GATEWAY_REPLICAS_DEFAULT=1 at L14. -- src/dstack/_internal/core/models/volumes.py — VolumeStatus (L19), BaseVolumeConfiguration (L36), AnyVolumeConfiguration (L179), VolumeConfiguration (L187), parse_volume_configuration (L191), VolumeSpec (L198), Volume (L233), VolumePlan (L287) — VolumeStatus: SUBMITTED, PROVISIONING (currently unused), ACTIVE, FAILED; finished_statuses() == [FAILED]. -- src/dstack/_internal/core/models/configurations.py — ApplyConfigurationType (L1384), AnyApplyConfiguration (L1393), BaseApplyConfiguration (L1401), parse_apply_configuration (L1424), AnyRunConfiguration (L1366) — Where a new top-level `type: endpoint` must be registered (enum value + union member + discriminator __root__). -- src/dstack/_internal/server/services/volumes.py — create_volume (L259), delete_volumes (L321), list_volumes (L104), get_volume_by_name (L222), volume_model_to_volume (L379), switch_volume_status (L54), emit_volume_status_change_event (L75), generate_volume_name (L457) — The cleanest single-file service blueprint for a new resource type. -- src/dstack/_internal/server/services/gateways/__init__.py — create_gateway (L224), delete_gateways (L335), get_plan (L926), apply_plan (L973), switch_gateway_status (L92), gateway_model_to_gateway (L879), get_gateway_configuration (L847), generate_gateway_name (L617) — Gateway service; includes the plan/apply (in-place update) pattern volumes lack. -- src/dstack/_internal/server/background/pipeline_tasks/base.py — PipelineItem (L34), Pipeline (L67), Heartbeater (L166), Fetcher (L257), Worker (L340), ItemUpdateMap (L403), NOW_PLACEHOLDER (L383), set_unlock_update_map_fields (L410), set_processed_update_map_fields (L416), resolve_now_placeholders (L430), log_lock_token_mismatch (L449), log_lock_token_changed_after_processing (L463) — Framework every new resource pipeline subclasses. Multi-replica safety = FOR UPDATE SKIP LOCKED fetch + lock_token lease + heartbeat renewal + conditional UPDATE on write-back. -- src/dstack/_internal/server/background/pipeline_tasks/volumes.py — VolumePipeline (L58), VolumeFetcher.fetch (L136), VolumeWorker.process (L213), _process_submitted_volume (L307), _process_to_be_deleted_volume (L374) — Simplest full pipeline example (~420 lines) — best copy-paste template for an EndpointPipeline. -- src/dstack/_internal/server/background/pipeline_tasks/gateways.py — GatewayPipeline (L59), GatewayFetcher.fetch (L137), GatewayWorker.process (L215), _process_submitted_gateway (L283), _process_provisioning_gateway (L380), _process_to_be_deleted_gateway (L475) — Example with an intermediate PROVISIONING state that polls child resources (gateway_computes) until all RUNNING; hard-deletes the row on deletion. -- src/dstack/_internal/server/background/pipeline_tasks/__init__.py — PipelineManager (L31), PipelineHinter (L80), get_pipeline_manager (L99), start_pipeline_tasks (L106) — New pipelines must be added to the PipelineManager.__init__ builtin list (L35-48). -- src/dstack/_internal/server/services/pipelines.py — PipelineHinterProtocol (L6), get_pipeline_hinter (L22) — FastAPI dependency giving handlers a hinter; create/apply endpoints call pipeline_hinter.hint_fetch(Model.__name__) after commit for low-latency pickup. -- src/dstack/_internal/server/routers/volumes.py — root_router + project_router; list/get/create/delete handlers — All handlers use Depends(ProjectMember()); create/delete return nothing special; ResourceNotExistsError raised for missing get. -- src/dstack/_internal/server/routers/gateways.py — router (prefix /api/project/{project_name}/gateways); get_plan (L74) + apply_plan (L96) use ProjectAdmin(); list uses ProjectMemberOrPublicAccess(); get uses Authenticated()+Project()+check_can_access_gateway — Registered in app.py L257-259 via app.include_router(). -- src/dstack/_internal/server/schemas/volumes.py — ListVolumesRequest, GetVolumeRequest, CreateVolumeRequest, DeleteVolumesRequest — Request body models; gateways version adds GetGatewayPlanRequest/ApplyGatewayPlanRequest. -- src/dstack/_internal/server/security/permissions.py — Authenticated (L37), GlobalAdmin (L49), ProjectAdmin (L63), ProjectManager (L84), ProjectMember (L112), ProjectMemberOrPublicAccess (L123), check_can_access_gateway (L315) — Exact dependency names. ProjectAdmin/ProjectMember __call__ returns Tuple[UserModel, ProjectModel]. -- src/dstack/_internal/cli/services/configurators/__init__.py — apply_configurators_mapping (L27), get_apply_configurator_class (L52), load_apply_configuration (L62) — New configurator class must be appended to the mapping list. -- src/dstack/_internal/cli/services/configurators/volume.py — VolumeConfigurator (TYPE=ApplyConfigurationType.VOLUME), apply_configuration, delete_configuration, register_args, apply_args — Client-side plan, delete-then-recreate on change, provisioning polling loop until ACTIVE/FAILED, exit(1) on FAILED. -- src/dstack/_internal/cli/services/configurators/gateway.py — GatewayConfigurator; uses server-side gateways.get_plan/apply_plan with legacy fallback — Polls until RUNNING/FAILED via _finished_provisioning (L284). -- src/dstack/api/server/__init__.py — APIClient.gateways property (L125) -> GatewaysAPIClient, APIClient.volumes (L129) -> VolumesAPIClient — Low-level client; per-resource group files _gateways.py/_volumes.py subclass APIClientGroup. -- src/dstack/_internal/server/services/events.py — emit (L171), Target.from_model (L89), SystemActor (L34), UserActor (L42), AnyActor (L61) — Target.from_model raises ValueError for unknown model types — a new resource model must be added to its Union + isinstance chain, plus EventTargetType in core/models/events.py (L12). -- src/dstack/_internal/server/services/locking.py — get_locker (L175), ResourceLocker.get_lockset (L37), ResourceLocker.lock_ctx (L46), string_to_lock_id (L121) — In-process/pg-advisory locking used for name-uniqueness during create and row lock acquisition during delete. -- src/dstack/_internal/server/testing/common.py — create_gateway (L639), create_gateway_compute (L683), create_volume (L1069) — Test factories; tests live in src/tests/_internal/server/background/pipeline_tasks/test_{volumes,gateways}.py and src/tests/_internal/server/routers/test_{volumes... (test_gateways.py exists; no test_volumes.py in routers dir listing — volumes router tests may be elsewhere). - -## Details -## 1. DB models (src/dstack/_internal/server/models.py) - -### PipelineModelMixin (models.py:204-207) -```python -class PipelineModelMixin: - lock_expires_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime) - lock_token: Mapped[Optional[uuid.UUID]] = mapped_column(UUIDType(binary=False)) - lock_owner: Mapped[Optional[str]] = mapped_column(String(100)) -``` -Every pipeline-processed model inherits this (GatewayModel, GatewayComputeModel, VolumeModel, FleetModel, PlacementGroupModel, ...). - -### GatewayModel (models.py:600-653), `__tablename__ = "gateways"` -- `id: Mapped[uuid.UUID] = mapped_column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4)` -- `name: String(100)`, `region: String(100)`, `wildcard_domain: Optional[String(100)]` -- `configuration: Mapped[Optional[str]] = mapped_column(Text)` — JSON of GatewayConfiguration (Optional only for pre-0.18.2 compat) -- `created_at: NaiveDateTime, default=get_current_datetime` -- `status: Mapped[GatewayStatus] = mapped_column(EnumAsString(GatewayStatus, 100))` (models.py:614) -- `status_message: Mapped[Optional[str]] = mapped_column(Text)` (615) -- `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime)` (616) -- `to_be_deleted: Mapped[bool] = mapped_column(Boolean, server_default=false())` (617) -- `project_id: ForeignKey("projects.id", ondelete="CASCADE")` + `project` relationship (624-625); `backend_id` FK likewise (626) -- `__table_args__ = (UniqueConstraint("project_id", "name", name="uq_gateways_project_id_name"),)` (651) -- NO `deleted` flag — gateways are hard-deleted (comment at 653: "TODO: Add pipeline index ... if gateways become soft-deleted"). - -### VolumeModel (models.py:951-998), `__tablename__ = "volumes"` -- `id` UUID pk as above; `name: String(100)` -- `user_id: ForeignKey("users.id", ondelete="CASCADE")` + `user` relationship (959-960) -- `project_id` FK CASCADE + `project` (962-963) -- `created_at`, `last_processed_at: NaiveDateTime, default=get_current_datetime` (965-968), `last_job_processed_at: Optional` (969) -- `deleted: Boolean, default=False` (973), `deleted_at: Optional[NaiveDateTime]` (974), `to_be_deleted: Boolean, server_default=false()` (975) — soft delete -- `status: mapped_column(EnumAsString(VolumeStatus, 100), index=True)` (977) with docstring "`status` must be changed only via `switch_volume_status()`" and `status_message: Text` (979) -- JSON-as-Text columns: `configuration` (981, required), `volume_provisioning_data` (982, optional) -- `attachments: relationship(VolumeAttachmentModel)` (986) -- Partial fetch index (991-998): -```python -Index("ix_volumes_pipeline_fetch_q", last_processed_at.asc(), - postgresql_where=deleted == false(), sqlite_where=deleted == false()) -``` - -### GatewayComputeModel (models.py:656-733) — child resource per gateway replica -Has its own `status: GatewayReplicaStatus`, `status_message`, `last_processed_at`, `deleted`, `ssh_private_key/ssh_public_key` (Text), `backend_data: Text`, and its own pipeline fetch index `ix_gateway_computes_pipeline_fetch_q` (727-732). - -## 2. Status enums and lifecycle - -`src/dstack/_internal/core/models/gateways.py:17-29`: -```python -class GatewayStatus(str, Enum): - SUBMITTED = "submitted"; PROVISIONING = "provisioning"; RUNNING = "running"; FAILED = "failed" -class GatewayReplicaStatus(str, Enum): - SUBMITTED, PROVISIONING, RUNNING, TERMINATING, TERMINATED -``` -`src/dstack/_internal/core/models/volumes.py:19-33`: -```python -class VolumeStatus(str, Enum): - SUBMITTED, PROVISIONING (unused), ACTIVE, FAILED - def is_active(self); @classmethod finished_statuses() -> [FAILED] -``` - -Lifecycle, volumes: API `create_volume` inserts row with `status=SUBMITTED` -> VolumeWorker `_process_submitted_volume` calls backend `compute.create_volume`/`register_volume` -> writes `status=ACTIVE, volume_provisioning_data=vpd.json()` or `status=FAILED, status_message=...`. There is no PROVISIONING step in practice. - -Lifecycle, gateways: `create_gateway` inserts `status=SUBMITTED` -> `_process_submitted_gateway` resolves backend, creates N GatewayComputeModel rows (SUBMITTED), sets gateway `status=PROVISIONING` -> the separate GatewayReplicaPipeline (`background/pipeline_tasks/gateway_replicas.py`) provisions each replica (`compute.create_gateway(compute_configuration)` at gateway_replicas.py:427, then connect + `gateways_services.configure_gateway(connection)` at :537) -> gateway pipeline `_process_provisioning_gateway` (gateways.py:380-401) polls replica statuses: any TERMINATING/TERMINATED => `FAILED` with `status_message="Failed to provision gateway replica"`; all RUNNING => `RUNNING`; otherwise no-op (stays PROVISIONING, gets re-fetched after min_processing_interval=15s). - -Deletion — asynchronous in both cases, user-facing call only marks the row: -- `delete_volumes` (services/volumes.py:321): selects rows, then under `get_locker(...).lock_ctx(VolumeModel.__tablename__, volumes_ids)` retries 10x to grab rows with `lock_expires_at IS NULL` FOR UPDATE; raises `ServerClientError("Failed to delete volumes: volumes are being processed currently. Try again later.")` if it can't; raises ServerClientError if attachments exist ("Volume is in use"); sets `to_be_deleted=True` + emits "Volume marked for deletion" event; commits. The pipeline `_process_to_be_deleted_volume` (pipeline_tasks/volumes.py:374) calls `compute.delete_volume` (errors logged, not retried) then writes `{deleted: True, deleted_at: NOW_PLACEHOLDER}` — soft delete. -- `delete_gateways` (services/gateways/__init__.py:335): same lock-retry pattern, sets `to_be_deleted=True`. GatewayWorker `_process_to_be_deleted_item` (pipeline_tasks/gateways.py:404) waits until all replica computes are TERMINATED (replica termination is driven by the GatewayReplicaPipeline) and then executes `delete(GatewayModel).where(id==..., lock_token==...)` — hard delete of the row, emitting "Gateway deleted" with `events.SystemActor()`. -- Additionally `process_idle_volumes` (background/scheduled_tasks/idle_volumes.py:25) sets `to_be_deleted = True` (line 92) for volumes idle past `auto_cleanup_duration` — an example of system-initiated deletion. - -## 3. Service modules - -### services/volumes.py -- `async def create_volume(session, project: ProjectModel, user: UserModel, configuration: AnyVolumeConfiguration, pipeline_hinter: PipelineHinterProtocol) -> Volume` (L259). Flow: `apply_plugin_policies(user=user.name, project=project.name, spec=VolumeSpec(configuration=configuration))` -> `_validate_volume_configuration` (raises `ServerClientError` for bad config / unsupported backend; `validate_dstack_resource_name` from `dstack._internal.core.services`) -> name-uniqueness critical section: `lock_namespace = f"volume_names_{project.name}"`; on sqlite `await session.commit()` first, on postgres `SELECT pg_advisory_xact_lock(string_to_lock_id(lock_namespace))`, plus in-process `get_locker(get_db().dialect_name).get_lockset(lock_namespace)`; duplicate name => `raise ResourceExistsError()`; auto-name via `generate_volume_name` (random_names.generate_name loop) -> insert model with `status=VolumeStatus.SUBMITTED, created_at=now, last_processed_at=now` -> `events.emit(...,"Volume created. Status: SUBMITTED", actor=events.UserActor.from_user(user), targets=[events.Target.from_model(volume_model)])` -> commit -> `pipeline_hinter.hint_fetch(VolumeModel.__name__)` (L317). -- `list_volumes(session, user, project_name, only_active, prev_created_at, prev_id, limit, ascending) -> List[Volume]` (L104) — keyset pagination on (created_at, id). -- `volume_model_to_volume(volume_model) -> Volume` (L379) — model->API conversion; JSON parsed via `VolumeConfiguration.__response__.parse_raw(...)`. -- `switch_volume_status(session, volume_model, new_status, actor=events.SystemActor())` (L54) — the only sanctioned in-session status setter, emits status-change event ("Volume status changed OLD -> NEW (message)"). - -### services/gateways/__init__.py -- `create_gateway(session, user, project, configuration: GatewayConfiguration, pipeline_hinter, *, effective_configuration=None) -> Gateway` (L224) — same plugin-policy/lock/name pattern (`lock_namespace = f"gateway_names_{project.name}"`); also resolves backend up-front with `get_project_backend_with_model_by_type_or_error` and handles default-gateway assignment; hints via `pipeline_hinter.hint_fetch(GatewayModel.__name__)` (L291). -- `get_plan(session, project, user, spec: GatewaySpec) -> GatewayPlan` (L926) — computes `ApplyAction.CREATE` vs `UPDATE` using `diff_models` (from `dstack._internal.core.services.diff`) and `_can_update_gateway_in_place` (L1050, checks diff keys against `_CONF_UPDATABLE_FIELDS`). -- `apply_plan(session, user, project, plan: ApplyGatewayPlanInput, force: bool, pipeline_hinter) -> Gateway` (L973) — creates if name unset/not found; rejects if `to_be_deleted` ("is being deleted. Try again later."); optimistic concurrency check unless `force` ("Failed to apply plan. Resource has been changed. Try again or use force apply."); in-place update writes configuration + emits "Gateway updated. Changed fields: ...". - -Error convention (src/dstack/_internal/core/errors.py): `ServerClientError` (L39) is the base 400-class error carrying `msg` and `code: ServerClientErrorCode`; subclasses `ResourceExistsError` (L52, msg "Resource exists"), `ResourceNotExistsError` (L57, "Resource not found"), `BackendNotAvailable` (L67), `GatewayError` (L80). Routers raise them directly; `get_base_api_additional_responses()` + exception handling in `server/utils/routers.py` (get_server_client_error_details L111) turn them into HTTP error bodies. `error_forbidden()/error_not_found()/error_invalid_token()` helpers are also in utils/routers.py (L83-104). - -## 4. Background processing - -Two mechanisms (see docstrings): -- `start_pipeline_tasks()` (background/pipeline_tasks/__init__.py:106): "fetch-workers pipelines based on db + in-memory queues ... tasks that run frequently and need to lock rows for a long time". -- `start_scheduled_tasks()` (background/scheduled_tasks/__init__.py:37): APScheduler `AsyncIOScheduler` for infrequent jobs (`process_gateways_connections` every 15s, `process_idle_volumes` every 60s, etc.). - -Both are started in the FastAPI lifespan in app.py (L178-181) guarded by `settings.SERVER_BACKGROUND_PROCESSING_ENABLED`, and `app.state.pipeline_manager = pipeline_manager` is what `get_pipeline_hinter` (services/pipelines.py:22) reads. Shutdown: `pipeline_manager.shutdown()` then `await pipeline_manager.drain()` (app.py:207-214). - -Pipeline anatomy (base.py): a `Pipeline` owns one `Fetcher`, N `Worker`s (default `workers_num=10`), one `Heartbeater`, and an `asyncio.Queue` sized `workers_num*2.0` max. Constructor defaults on VolumePipeline/GatewayPipeline: `min_processing_interval=15s, lock_timeout=30s, heartbeat_trigger=15s`. Required overrides: `hint_fetch_model_name` property (returns `Model.__name__`), `_heartbeater`, `_fetcher`, `_workers` properties. - -One VolumeFetcher.fetch iteration (pipeline_tasks/volumes.py:136-196), multi-replica safe: -1. In-process lockset for the table + session. -2. `select(VolumeModel).where(or_(status==SUBMITTED, to_be_deleted==True), deleted==False, or_(last_processed_at <= now - min_processing_interval, last_processed_at == created_at), or_(lock_expires_at.is_(None), lock_expires_at < now), or_(lock_owner.is_(None), lock_owner == VolumePipeline.__name__)).order_by(last_processed_at.asc()).limit(limit).with_for_update(skip_locked=True, key_share=True, of=VolumeModel)` with `load_only(...)`. -3. For each row set `lock_expires_at = now + lock_timeout`, `lock_token = uuid4()` (one token per batch), `lock_owner = VolumePipeline.__name__`; commit; return `VolumePipelineItem(__tablename__=..., id, lock_expires_at, lock_token, prev_lock_expired, status, to_be_deleted)` items which are queued and tracked by the Heartbeater (renews lock via conditional UPDATE while item is in flight). - -VolumeWorker.process (L213-226): refetch the row `WHERE id==item.id AND lock_token==item.lock_token` with joinedloads (`_refetch_locked_volume` L229); on mismatch `log_lock_token_mismatch` and drop. Dispatch: `to_be_deleted` -> `_process_to_be_deleted_volume`; `status==SUBMITTED` -> `_process_submitted_volume`. Result is a `_ProcessResult(update_map: _VolumeUpdateMap)` (TypedDict extending `ItemUpdateMap` with `status, status_message, volume_provisioning_data, deleted, deleted_at`). `_apply_process_result` (L249) merges, applies `set_processed_update_map_fields` (last_processed_at=NOW_PLACEHOLDER) + `set_unlock_update_map_fields` (nulls the 3 lock columns), resolves NOW placeholders, and executes `update(VolumeModel).where(id==..., lock_token==...).values(**update_map).returning(id)`; 0 rows => `log_lock_token_changed_after_processing` (state change is abandoned, will be reprocessed); else emits the delete/status-change event in the same transaction. - -Exception handling in workers: `Worker.start` (base.py:352-369) wraps `self.process(item)` in try/except logging "Unexpected exception when processing item" and always untracks from heartbeater — an unhandled exception leaves the row locked until `lock_expires_at` passes, then it is re-fetched. Inside `_process_submitted_volume`, `BackendNotAvailable` -> FAILED/"Backend not available"; `BackendError` -> FAILED with `str(e.args[0])` or `f"Backend error: {repr(e)}"`; bare `Exception` -> FAILED/`f"Unexpected error: {repr(e)}"` (L307-361). Sentry instrumentation via `@sentry_utils.instrument_pipeline_task("VolumeFetcher.fetch")` / `("VolumeWorker.process")`. - -Registration: add pipeline instance to the list in `PipelineManager.__init__` (pipeline_tasks/__init__.py:35-48). `PipelineHinter.register_pipeline` maps `hint_fetch_model_name` -> pipeline so API handlers can `hint_fetch("VolumeModel")`; unknown names just log a warning (L88-93). - -## 5. REST API - -Gateways router (routers/gateways.py): `router = APIRouter(prefix="/api/project/{project_name}/gateways", tags=["gateways"], responses=get_base_api_additional_responses())`. Endpoints: POST `/list` (ProjectMemberOrPublicAccess), `/get` (Authenticated + `Project()` dep from `server/deps.py:10` + `check_can_access_gateway` for cross-project imported gateways), `/get_plan` + `/apply` + `/create`(deprecated) + `/delete` + `/set_default` + `/set_wildcard_domain`(deprecated) — all mutating ones use `Depends(ProjectAdmin())`. `/apply` and `/create` take `pipeline_hinter: Annotated[PipelineHinterProtocol, Depends(get_pipeline_hinter)]`. Responses wrapped in `CustomORJSONResponse`; client-version compatibility patching via `patch_gateway/patch_gateway_plan` from `server/compatibility/gateways.py` keyed on `Depends(get_client_version)`. - -Volumes router (routers/volumes.py): TWO routers — `root_router = APIRouter(prefix="/api/volumes")` (global paginated `/list`, `Depends(Authenticated())`) and `project_router = APIRouter(prefix="/api/project/{project_name}/volumes")` (`/list`, `/get`, `/create`, `/delete` — ALL `Depends(ProjectMember())`, i.e. volumes need only member rights while gateways need admin). Both registered in `app.py` (L257-259): `app.include_router(gateways.router); app.include_router(volumes.root_router); app.include_router(volumes.project_router)`. - -Permission dependency exact names (security/permissions.py): `Authenticated` (L37, returns UserModel), `GlobalAdmin` (L49), `ProjectAdmin` (L63, returns Tuple[UserModel, ProjectModel]; global admins pass), `ProjectManager` (L84), `ProjectMember` (L112), `ProjectMemberOrPublicAccess` (L123), `ProjectManagerOrPublicProject` (L159), `ProjectManagerOrSelfLeave` (L195), `OptionalServiceAccount` (L236), plus `check_can_access_gateway` (L315). - -Schemas: `server/schemas/gateways.py` (CreateGatewayRequest, ListGatewaysRequest, GetGatewayRequest, GetGatewayPlanRequest, ApplyGatewayPlanRequest{plan, force}, DeleteGatewaysRequest{names}, SetDefaultGatewayRequest, SetWildcardDomainRequest); `server/schemas/volumes.py` (ListVolumesRequest{project_name, only_active, prev_created_at, prev_id, limit(<=100), ascending}, GetVolumeRequest{name}, CreateVolumeRequest{configuration: Annotated[AnyVolumeConfiguration, Field(discriminator="backend")]}, DeleteVolumesRequest{names}). All extend `CoreModel` from `core/models/common.py`. - -## 6. CLI - -`dstack apply -f x.yml` (cli/commands/apply.py `ApplyCommand`): `load_apply_configuration` (cli/services/configurators/__init__.py:62) parses YAML with `parse_apply_configuration` (core/models/configurations.py:1424), then `get_apply_configurator_class(configuration.type)` looks up `apply_configurators_mapping` (L27) = {DevEnvironmentConfigurator, TaskConfigurator, ServiceConfigurator, FleetConfigurator, GatewayConfigurator, VolumeConfigurator} keyed by `cls.TYPE: ApplyConfigurationType`. Configurator contract (`BaseApplyConfigurator` in configurators/base.py:20): ClassVar `TYPE`, `__init__(self, api_client: Client)`, abstract `apply_configuration(conf, configuration_path, command_args, configurator_args)` and `delete_configuration(conf, configuration_path, command_args)`, classmethods `get_parser()`/`register_args(parser)`. Flags handled by ApplyCommand itself: `-f/--file`, `-y/--yes`, `--force`, `-d/--detach`, `-v/--verbose`. - -`dstack delete -f x.yml` (cli/commands/delete.py `DeleteCommand`, ALIASES=["destroy"]) — same lookup, calls `configurator.delete_configuration(...)`. - -VolumeConfigurator.apply_configuration behavior: builds `VolumeSpec`, computes plan CLIENT-side (`_get_plan` at configurator/volume.py:174 — comment "TODO: Implement server-side /get_plan"), prints plan table, on config change deletes existing volume and polls `volumes.get` until `ResourceNotExistsError`, then `api.client.volumes.create(...)`, and unless `--detach` polls with `MultiItemStatus` every `LIVE_TABLE_PROVISION_INTERVAL_SECS` until status in [ACTIVE, FAILED]; on FAILED prints status_message and `exit(1)`; on Ctrl-C offers to delete. - -GatewayConfigurator uses server-side `gateways.get_plan`/`apply_plan` (with `MethodNotAllowedError` fallback to legacy create for pre-0.20.27 servers) and polls until RUNNING/FAILED. - -Management commands: `GatewayCommand` (cli/commands/gateway.py, NAME="gateway": subcommands list [default]/delete/update/get) and `VolumeCommand` (cli/commands/volume.py, NAME="volume": _list/_delete). Both registered in cli/main.py (`GatewayCommand.register(subparsers)` L74, `VolumeCommand.register(subparsers)` L86; `ApplyCommand` L67, `DeleteCommand` L69). - -## 7. Python API client (src/dstack/api/...) - -Low-level: `dstack/api/server/__init__.py` `class APIClient` with properties `gateways -> GatewaysAPIClient` (L125) and `volumes -> VolumesAPIClient` (L129). `dstack/api/server/_volumes.py` `VolumesAPIClient(APIClientGroup)` methods: `list(project_name)`, `get(project_name, name)`, `create(project_name, configuration)` (uses `get_create_volume_excludes` from `core/compatibility/volumes.py` to trim the body for old servers), `delete(project_name, names)`. `_gateways.py` `GatewaysAPIClient`: `list(project_name, *, include_imported=False)`, `get`, `get_plan(project_name, spec)`, `apply_plan(project_name, plan, *, force=False)`, `create`, `delete`, `set_default`, `set_wildcard_domain`. Responses parsed with `parse_obj_as(Model.__response__, resp.json())`. - -High-level: `dstack/api/_public/__init__.py` `Client` exposes only `runs`, `repos`, `backends` collections — there is NO high-level gateway/volume collection; the CLI reaches them via `self.api.client.gateways`/`self.api.client.volumes` (the raw `APIClient` behind `Client.client`). - -## 8. Supporting infrastructure verified - -- Events: `services/events.py` `emit(session, message, actor, targets)` (L171, adds to session without committing); `Target.from_model` (L89) accepts a closed Union of models and raises `ValueError(f"Unsupported model type: ...")` otherwise; `EventTargetType` enum in `core/models/events.py:12` (PROJECT/USER/FLEET/INSTANCE/RUN/JOB/VOLUME/GATEWAY/SECRET). A new resource requires: new EventTargetType member + new branch in Target.from_model. -- Plugins: `services/plugins.py` `apply_plugin_policies(user: str, project: str, spec: ApplySpec) -> ApplySpec` (L94); `ApplySpec = TypeVar("ApplySpec", RunSpec, FleetSpec, VolumeSpec, GatewaySpec)` in `dstack/plugins/_models.py:8` — a new spec type must be added to this TypeVar for plugin support. -- Locking: `services/locking.py` `get_locker(dialect_name) -> ResourceLocker` (L175); `get_lockset(namespace)`, `lock_ctx(namespace, keys)` (async ctx), `string_to_lock_id(s) -> int` (L121, for pg advisory locks). -- Backends: `services/backends/__init__.py` `get_project_backend_by_type_or_error` (L419), `get_project_backend_with_model_by_type_or_error` (L383), `check_backend_type_available` (L514). -- Migrations: alembic files under `src/dstack/_internal/server/migrations/versions//MM_DD_HHMM__.py`; pattern uses `op.batch_alter_table` (sqlite-compatible) and `dstack._internal.server.models.NaiveDateTime()`/`sqlalchemy_utils.types.uuid.UUIDType(binary=False)` column types (see 2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py). -- Tests: pipeline tests in `src/tests/_internal/server/background/pipeline_tasks/test_volumes.py`, `test_gateways.py`, `test_gateway_replicas.py`; router tests in `src/tests/_internal/server/routers/test_gateways.py`; factories `create_volume` (testing/common.py:1069), `create_gateway` (:639), `create_gateway_compute` (:683). - -## RECIPE: add a new resource type "endpoint" (derived from volumes, the simpler blueprint) - -1. **Core models** — new `src/dstack/_internal/core/models/endpoints.py`: `EndpointStatus(str, Enum)` (submitted/provisioning/active(or running)/failed), `EndpointConfiguration(CoreModel)` with `type: Literal["endpoint"] = "endpoint"`, `name: Optional[str]`, `model: str`, `env`, + inherit/embed ProfileParams (`core/models/profiles.py:310`) the way run configurations do, `EndpointSpec(CoreModel)` {configuration, configuration_path}, `Endpoint(CoreModel)` (API representation: id, name, project_name, user, configuration, created_at, status, status_message, ...), optional `EndpointPlan`. -2. **Register the config type** — `core/models/configurations.py`: add `ENDPOINT = "endpoint"` to `ApplyConfigurationType` (L1384); add `EndpointConfiguration` to `AnyApplyConfiguration` (L1393), `BaseApplyConfiguration.__root__` union (L1411), and `AnyDstackConfiguration` (L1440, feeds the JSON schema). -3. **DB model** — `server/models.py`: `class EndpointModel(PipelineModelMixin, BaseModel)` with UUID pk, name String(100), project FK CASCADE, user FK, `status EnumAsString(EndpointStatus, 100)`, `status_message Text`, `created_at`/`last_processed_at NaiveDateTime`, `to_be_deleted Boolean server_default=false()`, `deleted`/`deleted_at` if soft-delete (recommended, like volumes), `configuration Text` (JSON), any provisioning-data Text column, UniqueConstraint("project_id","name") or the volumes-style deleted==False partial index `ix_endpoints_pipeline_fetch_q` on `last_processed_at.asc()`. -4. **Alembic migration** — new file under `server/migrations/versions/2026/`, following the batch_alter_table/NaiveDateTime/UUIDType pattern. -5. **Service module** — `server/services/endpoints.py` copying services/volumes.py structure: `create_endpoint(session, project, user, configuration, pipeline_hinter)` (plugin policies -> validation raising ServerClientError -> `endpoint_names_{project.name}` lock -> ResourceExistsError on dup -> insert SUBMITTED -> events.emit -> commit -> `pipeline_hinter.hint_fetch(EndpointModel.__name__)`), `delete_endpoints(...)` (lock_ctx + set to_be_deleted), `list_/get_by_name`, `endpoint_model_to_endpoint`, `switch_endpoint_status`, `emit_endpoint_status_change_event`. -6. **Events** — `core/models/events.py` add `EventTargetType.ENDPOINT`; `server/services/events.py` add EndpointModel to `Target.from_model` union and isinstance chain. -7. **Pipeline** — `server/background/pipeline_tasks/endpoints.py` cloning volumes.py: `EndpointPipelineItem(PipelineItem)` {status, to_be_deleted}, `EndpointPipeline(Pipeline[...])` with `hint_fetch_model_name -> EndpointModel.__name__`, `EndpointFetcher.fetch` (SUBMITTED|PROVISIONING|to_be_deleted filter — include PROVISIONING like gateways since endpoint provisioning is long-running/polled), `EndpointWorker.process` dispatching `_process_submitted_endpoint` / `_process_provisioning_endpoint` / `_process_to_be_deleted_endpoint`, `_EndpointUpdateMap(ItemUpdateMap)`, write-back via conditional UPDATE + `set_processed_update_map_fields`/`set_unlock_update_map_fields`. Register in `PipelineManager.__init__` list (background/pipeline_tasks/__init__.py:35-48). NOTE: worker `lock_timeout=30s` with heartbeat renewal means long operations (LLM agent run) are fine while the worker holds the item, since the Heartbeater renews the lease — but the whole operation blocks one of the 10 workers; the gateway pattern (state machine with short steps polled every 15s, child resources doing the long work) fits an agent-driven flow better. -8. **Schemas + router** — `server/schemas/endpoints.py` (Create/Get/List/Delete request models) and `server/routers/endpoints.py` with `project_router = APIRouter(prefix="/api/project/{project_name}/endpoints", tags=["endpoints"], responses=get_base_api_additional_responses())`; use `Depends(ProjectMember())` (volumes precedent) or `Depends(ProjectAdmin())` (gateways precedent) and `Depends(get_pipeline_hinter)` on create; register in `app.py` `create_app()` next to L257-259. -9. **API client** — `dstack/api/server/_endpoints.py` `EndpointsAPIClient(APIClientGroup)` with list/get/create/delete; add property to `APIClient` in `dstack/api/server/__init__.py`. -10. **CLI** — `cli/services/configurators/endpoint.py` `EndpointConfigurator(BaseApplyConfigurator[EndpointConfiguration])` with `TYPE = ApplyConfigurationType.ENDPOINT`, apply_configuration (plan print, create, poll until ACTIVE/FAILED, exit(1) on FAILED) and delete_configuration; add to `apply_configurators_mapping` in `cli/services/configurators/__init__.py:27`. Optionally `cli/commands/endpoint.py` `EndpointCommand` (list/delete) registered in `cli/main.py`. -11. **Plugins (optional)** — add `EndpointSpec` to the `ApplySpec` TypeVar in `dstack/plugins/_models.py:8` if `apply_plugin_policies` should run on endpoint specs (or skip plugin policies initially — but then don't call `apply_plugin_policies` in the service, since it's typed against that TypeVar). -12. **Tests** — factory `create_endpoint` in `server/testing/common.py`; pipeline tests in `src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py`; router tests in `src/tests/_internal/server/routers/`. - -## Gotchas -1) The old `background/tasks/process_*` layout is gone — processing now lives in `background/pipeline_tasks/` (Pipeline/Fetcher/Worker/Heartbeater classes) and `background/scheduled_tasks/` (APScheduler). Do not plan APIs like `process_submitted_volumes()` as a scheduled job; new resources get a Pipeline subclass registered in `PipelineManager.__init__`. -2) Status must NOT be written by assigning `model.status` in API handlers except via `switch_volume_status`/`switch_gateway_status` (VolumeModel.status docstring says so); pipelines never mutate the ORM object for final state — they write through `update(...).where(id==..., lock_token==...)` with an ItemUpdateMap so a stolen lock aborts the write. A new pipeline must copy this pattern exactly (including `set_unlock_update_map_fields` + `set_processed_update_map_fields` + `resolve_now_placeholders`). -3) Deletion is asynchronous and marker-based: API sets `to_be_deleted=True` only after acquiring row locks with a 10-retry loop and raises ServerClientError if rows are currently locked by a pipeline (`lock_expires_at IS NOT NULL`). Volumes soft-delete (`deleted`, `deleted_at`); gateways HARD-delete the row (`delete(GatewayModel)`) — gateways have no `deleted` column. Pick one consciously; the volumes fetch query and index both filter `deleted == False`. -4) `pipeline_hinter.hint_fetch()` takes the model CLASS name string (`VolumeModel.__name__`), not the table name, and only hints pipelines on the same server replica; correctness relies on the fetcher's polling (fetch_delays 0.5-5s + min_processing_interval), the hint is just latency optimization. Routers get it via `Depends(get_pipeline_hinter)`; background code can use `get_pipeline_manager().hinter`. -5) The fetch query has subtle conditions to copy verbatim: `or_(last_processed_at <= now - min_processing_interval, last_processed_at == created_at)` (process new items immediately), `lock_owner IS NULL OR lock_owner == .__name__` (lets other components take exclusive ownership), and `with_for_update(skip_locked=True, key_share=True, of=Model)`. Also `created_at` must equal `last_processed_at` on insert for the immediate-processing trick to work (services set both to the same `now`). -6) Worker `lock_timeout` is 30s but the Heartbeater renews leases every ~15s while an item is being processed, so long-running work in a worker is safe; however an unhandled exception leaves the row locked until expiry (up to 30s delay before retry) and the item is simply re-fetched — process functions are expected to be idempotent/retryable. -7) `events.emit()` does NOT commit; it must be called inside the transaction that commits the change. `Target.from_model` raises ValueError for unregistered model types — adding a resource without touching events.py + EventTargetType will crash the first emit. -8) Permission choice differs between the two blueprints: volumes use `ProjectMember()` for create/delete; gateways use `ProjectAdmin()` for everything mutating. Both return `Tuple[UserModel, ProjectModel]`. Names like `ProjectUser` or `ProjectRead` do not exist. -9) Name-uniqueness on create uses a three-part dance: sqlite -> `await session.commit()` first; postgres -> `pg_advisory_xact_lock(string_to_lock_id(ns))`; plus the in-process lockset. Volumes additionally rely on this instead of a DB unique constraint (VolumeModel has NO unique constraint on (project_id, name) because of soft-deleted rows; GatewayModel HAS `uq_gateways_project_id_name`). -10) `parse_apply_configuration` does two-pass parsing: extras-tolerant `BaseApplyConfiguration.__response__` first, then strict re-parse; volume configs re-dispatch on the `backend` discriminator. A single flat EndpointConfiguration is a "final configuration" and needs no second pass, but it must be added to BOTH `AnyApplyConfiguration` and `BaseApplyConfiguration.__root__`. -11) The high-level `dstack.api._public.Client` deliberately exposes only runs/repos/backends; CLI configurators use `self.api.client.` (raw APIClient). Don't plan a `Client.endpoints` collection as "the pattern" — the pattern is an APIClientGroup on the low-level client. -12) `apply_plugin_policies` is typed against `ApplySpec = TypeVar(..., RunSpec, FleetSpec, VolumeSpec, GatewaySpec)` (dstack/plugins/_models.py:8) — calling it with a new spec type without extending the TypeVar breaks the plugin contract (and type checking). -13) Volume plan is computed client-side in the CLI (no /get_plan endpoint for volumes); gateways have server-side get_plan/apply_plan with optimistic-concurrency `current_resource` checking and in-place-update whitelisting (`_can_update_gateway_in_place`). For a new resource, the gateway plan/apply flow is the more modern pattern. -14) `SERVER_BACKGROUND_PROCESSING_ENABLED` can be off (app.py:178) — routers must tolerate a missing pipeline_manager (`get_pipeline_hinter` returns a no-op hinter), i.e., never assume processing happens in-request. -15) The two-level gateway design (GatewayModel + GatewayComputeModel children, each with its own pipeline; parent polls child statuses in its PROVISIONING handler) is the precedent for an endpoint that drives a long provisioning process (e.g., a service run created by an agent): the parent state machine stays cheap and re-entrant, re-processed at most every `min_processing_interval` (15s). diff --git a/endpoint-implementation-research/run-submission-internals.md b/endpoint-implementation-research/run-submission-internals.md deleted file mode 100644 index 12e4233e0c..0000000000 --- a/endpoint-implementation-research/run-submission-internals.md +++ /dev/null @@ -1,126 +0,0 @@ -# run-submission-internals - -## Summary -Server-side run creation is fully feasible from a background task: `dstack._internal.server.services.runs` (now a package at src/dstack/_internal/server/services/runs/) exposes `get_plan`, `apply_plan`, `submit_run`, `stop_runs`, `delete_runs`, and `get_run_by_name/-id` that take only (AsyncSession, UserModel, ProjectModel, RunSpec/plan) — no HTTP request context. Repo-less runs are first-class: leaving `run_spec.repo_id`/`repo_data` as None makes the server default to the virtual repo (id "none") and auto-create the RepoModel row; no code upload is needed. There is NO existing precedent of the server creating whole runs from background tasks (submit_run/apply_plan are only called from routers/runs.py), but the server does create replacement/scale-up JobModels in the RunPipeline (rolling deployments, retries, scheduled runs), and the run/job/instance state machines are driven entirely by the multi-replica-safe pipeline_tasks framework (row locks via lock_owner/lock_token/lock_expires_at + FOR UPDATE SKIP LOCKED). There is no dedicated system user; the closest is the startup-created "admin" user (`get_or_create_admin_user`), and UserModel stores an encrypted API token plus per-user SSH keypair (required for run submission). RunModel has no tags/labels column — linking an endpoint to its run must be done via run_name convention or an FK on the new endpoint table. - -## Key files -- src/dstack/_internal/server/services/runs/__init__.py — get_plan, apply_plan, submit_run, stop_runs, delete_runs, get_run, get_run_by_name, get_run_by_id, get_run_model_by_name, run_model_to_run, switch_run_status, create_job_model_for_new_submission, is_job_ready, _generate_run_name, _get_run_repo_or_error — The whole server-side run lifecycle API. All functions take (session, user/project models, RunSpec) — callable from background tasks. -- src/dstack/_internal/server/services/runs/spec.py — validate_run_spec_and_set_defaults, check_can_update_run_spec, can_update_run_spec — Sets virtual-repo defaults (repo_id='none', VirtualRunRepoData) and requires ssh_key_pub or user.ssh_public_key. -- src/dstack/_internal/core/models/runs.py — RunSpec (l.522), RunStatus (l.652), RunTerminationReason (l.91), JobStatus (l.62), JobTerminationReason (l.134), Run (l.675), RunPlan (l.715), ApplyRunPlanInput (l.730), ServiceSpec (l.643) — Core pydantic models; RunStatus.finished_statuses() = [TERMINATED, FAILED, DONE]. -- src/dstack/_internal/core/models/repos/virtual.py — DEFAULT_VIRTUAL_REPO_ID='none', VirtualRepoInfo, VirtualRunRepoData, VirtualRepo — Virtual (repo-less) repo mechanism. -- src/dstack/_internal/server/routers/runs.py — root_router /api/runs/list; project_router /api/project/{project_name}/runs/{get,get_plan,apply,stop,delete,submit(deprecated)} — The only place submit_run/apply_plan/stop_runs are invoked today; shows required call pattern incl. pipeline_hinter and ssh-key refresh. -- src/dstack/_internal/server/schemas/runs.py — GetRunPlanRequest, ApplyRunPlanRequest, StopRunsRequest, SubmitRunRequest — API request schemas. -- src/dstack/_internal/server/background/pipeline_tasks/runs/__init__.py — RunPipeline, RunFetcher, RunWorker — Multi-replica-safe processing pattern (lock_token/lock_expires_at/lock_owner + FOR UPDATE SKIP LOCKED) to copy for an EndpointPipeline. -- src/dstack/_internal/server/background/pipeline_tasks/runs/active.py — process_active_run, _get_active_run_transition, _analyze_active_run — Run status semantics: RUNNING if any replica job RUNNING; FAILED via TERMINATING+termination_reason; rolling deployment/scaling create JobModels here (server-side job creation precedent). -- src/dstack/_internal/server/background/pipeline_tasks/__init__.py — PipelineManager, PipelineHinter, start_pipeline_tasks, get_pipeline_manager — Where a new pipeline would be registered; started from app.py lifespan when SERVER_BACKGROUND_PROCESSING_ENABLED. -- src/dstack/_internal/server/services/services/__init__.py — register_service, _register_service_in_server, _register_service_in_gateway — Called synchronously inside submit_run for service runs; produces ServiceSpec url (/proxy/services/// or gateway URL) and model mapping. -- src/dstack/_internal/server/services/users.py — get_or_create_admin_user (l.45), create_user (l.160), get_user_model_by_name (l.327), refresh_ssh_key (l.232) — Identity options for a background task; users get RSA ssh keypair on creation; token stored encrypted. -- src/dstack/_internal/server/services/projects.py — get_project_model_by_name (l.572), get_project_model_by_name_or_error (l.589) — How to load ProjectModel (with backends+members joined) to act 'as' a project. -- src/dstack/_internal/server/models.py — RunModel (l.405), UserModel (l.210), PipelineModelMixin (l.204) — RunModel columns — no tags/labels column; UserModel.token is EncryptedString + token_hash. -- src/dstack/_internal/server/services/repos.py — create_or_update_repo (l.101), get_repo_model (l.317), get_code_model (l.331) — Virtual repo row auto-created by _get_run_repo_or_error during submit_run. -- src/dstack/api/_public/runs.py — RunCollection.get_run_plan (l.470), apply_plan (l.547), apply_configuration (l.585) — Client-side reference for how RunSpec is constructed for repo-less configs. -- src/dstack/_internal/cli/services/configurators/run.py — BaseRunConfigurator.apply_configuration (l.87), ServiceConfigurator (l.716) — CLI apply flow for services; falls back to init_default_virtual_repo when no repo. - -## Details -# Ground truth: server-side run (service) creation, monitoring, termination - -## 1. services/runs package (src/dstack/_internal/server/services/runs/) -Package files: `__init__.py` (1243 l.), `plan.py`, `replicas.py`, `spec.py`, `router_worker_sync.py`, `service_router_worker_sync.py`. - -Exact signatures (all in `src/dstack/_internal/server/services/runs/__init__.py`): - -- `async def get_plan(session: AsyncSession, project: ProjectModel, user: UserModel, run_spec: RunSpec, max_offers: Optional[int], legacy_repo_dir: bool = False) -> RunPlan` (l.528). Applies plugin policies, validates spec, computes `job_plans` via `get_job_plans` (runs/plan.py), detects `action` CREATE vs UPDATE. Optional step — NOT required before submit. -- `async def apply_plan(session: AsyncSession, user: UserModel, project: ProjectModel, plan: ApplyRunPlanInput, force: bool, pipeline_hinter: Optional[PipelineHinterProtocol] = None, legacy_repo_dir: bool = False) -> Run` (l.587). Applies plugin policies; if `run_spec.run_name is None` or no active run with that name → delegates to `submit_run`; else attempts in-place update (only fields in `_UPDATABLE_SPEC_FIELDS`/`_CONF_UPDATABLE_FIELDS`, spec.py:26-63), bumping `deployment_num` (rolling deployment for services). -- `async def submit_run(session: AsyncSession, user: UserModel, project: ProjectModel, run_spec: RunSpec, pipeline_hinter: Optional[PipelineHinterProtocol] = None) -> Run` (l.681). Flow: `validate_run_spec_and_set_defaults(user, run_spec)` → `_get_run_repo_or_error` (auto-creates virtual repo) → `get_project_secrets_mapping` → run-name lock (`pg_advisory_xact_lock` / lockset namespace `run_names_{project.name}`) → generate name if None (`_generate_run_name`, l.1134, adjective-animal-N) else `delete_runs` of finished same-name run → `_validate_run` (volumes) → creates `RunModel(...)` (l.734: initial_status=SUBMITTED, or PENDING+0 replicas if `merged_profile.schedule`; `desired_replica_count=1` — real value set by RunPipeline; `priority=run_spec.configuration.priority`; `deployment_num=0`) → for services: `await services.register_service(session, run_model, run_spec)` then per replica-group creates jobs via `get_jobs_from_run_spec(run_spec=..., secrets=..., replica_num=..., replica_group_name=...)` and `create_job_model_for_new_submission(run_model, job, status=JobStatus.SUBMITTED)` (l.833), plus `ensure_service_router_worker_sync_row(session, run_model, run_spec)` → `session.commit()` → `pipeline_hinter.hint_fetch("JobModel"/"RunModel")` if provided → returns `await get_run_by_id(...)`. -- `async def stop_runs(session: AsyncSession, user: UserModel, project: ProjectModel, runs_names: List[str], abort: bool, pipeline_hinter: Optional[PipelineHinterProtocol] = None)` (l.865). Sets `termination_reason = RunTerminationReason.ABORTED_BY_USER|STOPPED_BY_USER`, `switch_run_status(session, run_model, RunStatus.TERMINATING, actor=UserActor)`, `skip_min_processing_interval=True`; the RunPipeline finishes termination. Commits. -- `async def delete_runs(session, user, project, runs_names: List[str])` (l.909) — only finished runs; sets `deleted=True`. -- Reads: `async def get_run(session, project, run_name=None, run_id=None) -> Optional[Run]` (l.456); `get_run_model_by_name(session, project, run_name) -> Optional[RunModel]` (l.477); `get_run_by_name` (l.496); `get_run_by_id` (l.507); `def run_model_to_run(run_model, include_jobs=True, job_submissions_limit=None, return_in_api=False, include_sensitive=False, include_job_connection_info=False, loaded_jobs=None) -> Run` (l.949) — populates `run.service` from `run_model.service_spec` JSON. -- `def switch_run_status(session, run_model, new_status: RunStatus, actor: events.AnyActor = events.SystemActor())` (l.97) — the ONLY sanctioned way to change RunModel.status (emits event). -- `def get_run_spec(run_model) -> RunSpec` (l.154) — parses `run_model.run_spec` JSON. -- `def is_job_ready(probes: Iterable[ProbeModel], probe_specs: Iterable[ProbeSpec]) -> bool` (l.1226). - -spec.py: `def validate_run_spec_and_set_defaults(user: UserModel, run_spec: RunSpec, legacy_repo_dir: bool = False)` (l.66): validates run_name against `^[a-z][a-z0-9-]{1,40}$`; sets `repo_id=DEFAULT_VIRTUAL_REPO_ID`/`repo_data=VirtualRunRepoData()` when both None (l.88-91); sets priority default; requires ssh key (l.128-132: `raise ServerClientError("ssh_key_pub must be set if the user has no ssh_public_key")`). - -## 2. Repo requirement / virtual repo -`src/dstack/_internal/core/models/repos/virtual.py`: `DEFAULT_VIRTUAL_REPO_ID = "none"` (l.11), `class VirtualRepoInfo(BaseRepoInfo)` with `repo_type: Literal["virtual"]`, `class VirtualRunRepoData(VirtualRepoInfo)`, `class VirtualRepo(Repo)` (programmatic files via `add_file`, tarred by `write_code_file`). - -Server mechanism: a run CAN be submitted with `run_spec.repo_id=None, repo_data=None`; `validate_run_spec_and_set_defaults` fills the virtual defaults, then `_get_run_repo_or_error` (runs/__init__.py:1185) sees `repo_data.repo_type == "virtual"` and calls `repos_services.create_or_update_repo(session, project, repo_id, repo_info)` (repos.py:101) which upserts the RepoModel row — i.e. NO prior `/repos/init` call is needed server-side. If `repo_code_hash` is None, no code blob is fetched for the runner (`_get_job_code` in background/pipeline_tasks/jobs_running.py:1745 returns None when `code_hash is None`). - -CLI behavior for repo-less configs: `BaseRunConfigurator.apply_configuration` (cli/services/configurators/run.py:112-114) — if `self.get_repo(...)` returns None it calls `init_default_virtual_repo(api)` (cli/services/repos.py:34-37: `repo = VirtualRepo(); api.repos.init(repo)`). The SDK (`RunCollection.get_run_plan`, api/_public/runs.py:498-542) builds `RunSpec(run_name=configuration.name, repo_id=repo.repo_id, repo_data=repo.run_repo_data, repo_code_hash=None-if-no-files, file_archives=..., configuration=..., profile=..., ssh_key_pub=None → server-managed user key)`. - -## 3. dstack apply flow for a service -CLI: `ServiceConfigurator` (cli/services/configurators/run.py:716) extends `BaseRunConfigurator.apply_configuration` (l.87) → `self.api.runs.get_run_plan(configuration, repo, configuration_path, profile, ssh_identity_file)` (api/_public/runs.py:470) → HTTP `POST /api/project/{project_name}/runs/get_plan` → router `get_plan` (server/routers/runs.py:112-140; body `GetRunPlanRequest{run_spec: RunSpec, max_offers: Optional[int]}`; deps `ProjectMember()`, `get_client_version`, `use_legacy_repo_dir`) → `runs.get_plan(...)`. Then `self.api.runs.apply_plan(run_plan, repo, reserve_ports)` (api/_public/runs.py:547; uploads code tar via `repos.upload_code` only if `repo.has_code_to_write()`) → `POST /api/project/{project_name}/runs/apply` (routers/runs.py:143-171; body `ApplyRunPlanRequest{plan: ApplyRunPlanInput{run_spec, current_resource}, force: bool}`) → `runs.apply_plan(...)`. Router also calls `users.refresh_ssh_key(session, actor=user)` when user has no ssh key. Other endpoints: `POST /api/runs/list` (root router), `POST /api/project/{p}/runs/get`, `/stop` (`StopRunsRequest{runs_names, abort}`), `/delete`, `/submit` (deprecated, body `SubmitRunRequest{run_spec}`). - -## 4. Stop/terminate + status semantics -Terminate programmatically: `runs.stop_runs(session, user, project, runs_names, abort, pipeline_hinter=None)` — this is the idiomatic way (routers use it); it flips status to TERMINATING and RunPipeline (`_process_terminating_item` → `process_terminating_run` in background/pipeline_tasks/runs/terminating.py:61) unregisters the service from the gateway (`_unregister_service`, terminating.py:146) and terminates jobs. - -`RunStatus` (core/models/runs.py:652): PENDING, SUBMITTED, PROVISIONING, RUNNING, TERMINATING, TERMINATED, FAILED, DONE; `finished_statuses() = [TERMINATED, FAILED, DONE]`; `is_finished()`. `RunTerminationReason` (l.91): ALL_JOBS_DONE→DONE, JOB_FAILED→FAILED, RETRY_LIMIT_EXCEEDED→FAILED, STOPPED_BY_USER→TERMINATED, ABORTED_BY_USER→TERMINATED, SERVER_ERROR→FAILED (`to_status()`, l.114). - -Service semantics (background/pipeline_tasks/runs/active.py `_get_active_run_transition`, l.369): run is RUNNING when ANY replica contributes RUNNING (job RUNNING); PROVISIONING when best is PROVISIONING/PULLING; FAILED (via TERMINATING + JOB_FAILED/RETRY_LIMIT_EXCEEDED) when a replica failed non-retryably; PENDING when all replicas are retrying. `Run.error` = `termination_reason.to_error()`; `Run.status_message` may show "pulling"/"retrying" (runs/__init__.py:1079). - -"Ready" for services is stronger than RUNNING: a replica is registered on the gateway/in-server proxy only when its probes are all ready — `_maybe_register_replica` (jobs_running.py:1119-1130) gates on `is_job_ready(job_model.probes, job_spec.probes)`; `is_probe_ready(probe, spec) = probe.success_streak >= spec.ready_after` (services/probes.py:9). If `ServiceConfiguration.model` is set and `probes` omitted, a default `/v1/chat/completions` probe is applied (configurations.py:1019-1026). Service URL: `Run.service: Optional[ServiceSpec]` — for in-server proxy `url = /proxy/services/{project}/{run_name}/`, model at `/proxy/models/{project}/` (services/services/__init__.py:240-282, `_register_service_in_server`); ServiceSpec.model = `ServiceModelSpec(name, base_url, type)`. - -## 5. Precedent for server-created runs -`grep submit_run(` over src: only routers/runs.py:217 and internal recursion in apply_plan (runs/__init__.py:608,621). `RunModel(` is instantiated exactly once in prod code (runs/__init__.py:734). So: NO existing code path where the server creates a run without a user API call. Closest precedents (server creates JOBS, not runs): (a) retry — `_build_retry_job_models` (active.py:455); (b) service auto-scaling — `_build_service_scaling_maps` (active.py:576) / `build_scale_up_job_models` (pipeline_tasks/runs/common.py:56); (c) rolling deployment — `_build_rolling_deployment_maps` (active.py:645); (d) scheduled runs — run stays PENDING and RunFetcher picks it up when `next_triggered_at < now` (pipeline_tasks/runs/__init__.py:157-163), then `process_pending_run` (pending.py:43) creates job models. All of these keep the user-created RunModel and attribute job creation to `events.SystemActor()` (see comment at runs/__init__.py:814-817). - -## 6. Identity for a background task -- No system/global service user exists. `get_or_create_admin_user(session) -> Tuple[UserModel, bool]` (services/users.py:45; username "admin", GlobalRole.ADMIN, token from `DSTACK_SERVER_ADMIN_TOKEN` or uuid4) is created at startup (app.py:142) and is the only guaranteed user. -- `UserModel.token: Mapped[DecryptedString] = mapped_column(EncryptedString(200), unique=True)` + `token_hash` (models.py:218-220); plaintext via `admin.token.get_plaintext_or_error()` (used in app.py:166,198). ssh_private_key/ssh_public_key columns nullable (pre-0.19.33 users); `create_user` (users.py:160) always generates an RSA pair. -- Acting "as" a project: load `ProjectModel` via `projects.get_project_model_by_name_or_error(session, project_name)` (services/projects.py:589; joinedloads backends+members) and pass it straight to the runs service functions — the service layer does no auth (auth is only `ProjectMember()` in security/permissions.py:112, a FastAPI dep resolving Bearer token → (UserModel, ProjectModel)). Idiomatic choice for an endpoint feature: store the endpoint creator's `user_id` on the endpoint model and submit runs as that user (events/attribution stay correct); events emitted by the background machinery itself use `events.SystemActor()` (services/events.py:34; `AnyActor = Union[SystemActor, UserActor]`, `emit(session, message, actor, targets)` l.171). -- Multi-replica-safe background patterns: (a) Pipeline framework — subclass `Pipeline[ItemT]`/`Fetcher`/`Worker`/`Heartbeater` (background/pipeline_tasks/base.py; items carry lock_token/lock_expires_at; fetch uses `.with_for_update(skip_locked=True, key_share=True)` + `lock_owner` — see RunFetcher.fetch, pipeline_tasks/runs/__init__.py:132-233), register in `PipelineManager.__init__` (pipeline_tasks/__init__.py:35-49), model gets `PipelineModelMixin` columns (models.py:204-207); or (b) apscheduler scheduled task — add in `start_scheduled_tasks()` (background/scheduled_tasks/__init__.py:37). Both started in app.py lifespan iff `settings.SERVER_BACKGROUND_PROCESSING_ENABLED` (app.py:178-183). DB sessions in background code come from `get_session_ctx()` (server/db.py). - -## 7. Naming and tagging -- Name rule: `validate_dstack_resource_name` → regex `^[a-z][a-z0-9-]{1,40}$` (core/services/__init__.py:6-12). Auto-generation: `generate_name()` (utils/random_names.py:253, adjective-animal) + `-{idx}` uniqueness loop scoped to project + non-deleted (`_generate_run_name`, runs/__init__.py:1134). -- Uniqueness: enforced among non-deleted runs per project under an advisory lock; submitting with an existing name of a FINISHED run marks the old run deleted (submit_run l.716-718); an ACTIVE same-name run makes apply_plan try in-place update or raise "Cannot override active run". -- Tags/labels: RunModel (models.py:405-465) has NO tag column. `ProfileParams.tags: Optional[Dict[str,str]]` (core/models/profiles.py:462) rides inside run_spec JSON and is propagated to backend cloud resources — not SQL-filterable. To link runs to an owning endpoint: use a deterministic run_name (e.g. derived from endpoint name) and/or an FK from the new EndpointModel to `RunModel.id`; RunModel.fleet_id (models.py:422) is prior art for run→resource attachment. - -## Minimal server-side call sequence (verified building blocks) -```python -from dstack._internal.server.db import get_session_ctx -from dstack._internal.server.services import projects as projects_services -from dstack._internal.server.services import runs as runs_services -from dstack._internal.server.services import users as users_services -from dstack._internal.core.models.runs import RunSpec, RunStatus -from dstack._internal.core.models.configurations import ServiceConfiguration - -async with get_session_ctx() as session: - project = await projects_services.get_project_model_by_name_or_error(session, project_name) - user = await users_services.get_user_model_by_name(session, username) # e.g. endpoint creator or "admin" - run_spec = RunSpec( - run_name="my-endpoint-svc", # or None -> auto-generated - configuration=ServiceConfiguration(commands=[...], port=8000, model="", env=..., replicas=...), - # repo_id / repo_data left None -> virtual repo defaults applied server-side - ssh_key_pub=None, # ok iff user.ssh_public_key is set - ) - run = await runs_services.submit_run(session=session, user=user, project=project, - run_spec=run_spec, pipeline_hinter=None) # commits - -# monitor (poll): -async with get_session_ctx() as session: - project = await projects_services.get_project_model_by_name_or_error(session, project_name) - run = await runs_services.get_run_by_name(session=session, project=project, run_name=name) - # run.status (RunStatus), run.error, run.service.url / run.service.model.base_url - # readiness: run.status == RunStatus.RUNNING (+ probe-based gateway registration already gated server-side) - -# terminate: -async with get_session_ctx() as session: - await runs_services.stop_runs(session=session, user=user, project=project, - runs_names=[name], abort=False, pipeline_hinter=None) # commits -``` -Testing helpers exist: `create_user`, `create_project`, `create_repo`, `create_run(session, project, repo, user, run_name=None, status=RunStatus.SUBMITTED, run_spec=None, ...) -> RunModel` in src/dstack/_internal/server/testing/common.py (create_run at l.365). - -Not verified/does not exist: no `runs/apply.py` module (apply logic lives in runs/__init__.py); no server-side "preset service config" store; no endpoint-like resource today; no system user; no run tag columns. - -## Gotchas -1) `submit_run` COMMITS the session multiple times (also via its internal `delete_runs` call) and uses cross-process locking (`pg_advisory_xact_lock` on Postgres / in-memory lockset on SQLite) for run-name uniqueness — call it with a fresh session from `get_session_ctx()` in a background task, never inside another transaction you expect to control. -2) `validate_run_spec_and_set_defaults` (called by both apply_plan and submit_run) raises ServerClientError unless `run_spec.ssh_key_pub` is set OR `user.ssh_public_key` is non-empty (spec.py:128-132). Users created via `create_user` always have keys; users created via testing helpers or pre-0.19.33 may not. -3) `submit_run` does NOT apply plugin policies — only `get_plan`/`apply_plan` call `apply_plugin_policies`. If the endpoint feature should respect plugins, go through `apply_plan(plan=ApplyRunPlanInput(run_spec=...), force=True)` rather than `submit_run` directly. Note `apply_plan` with `run_spec.run_name=None` just delegates to submit_run. -4) For services, `submit_run` synchronously calls `services.register_service` (runs/__init__.py:758-760, marked FIXME) — it can raise ResourceNotExistsError (gateway referenced but missing, or `FORBID_SERVICES_WITHOUT_GATEWAY` set) or ServerClientError (autoscaling/rate_limits without gateway). An endpoint background task must handle these as submission failures. -5) No membership/authorization checks exist in the service layer — `ProjectMember()` in routers is the only gate. A background task passing any UserModel works; the run is attributed to that user (RunModel.user_id, events UserActor). -6) There is NO system user. Only the `admin` user (username "admin", GlobalRole.ADMIN) is guaranteed to exist (created in app.py:142 at startup). If endpoints should be attributed to the submitting user, store the endpoint's creator user_id and pass that UserModel. -7) RunModel has NO tags column. `ProfileParams.tags` (profiles.py:462) exists in run_spec JSON only (propagated to cloud resources, not SQL-queryable). Link endpoint→run via run_name (regex `^[a-z][a-z0-9-]{1,40}$`, unique among non-deleted runs per project) or an FK to RunModel.id on the new endpoint table. -8) Run status is not a reliable "ready" signal for services: RunStatus.RUNNING means ≥1 replica job is RUNNING, but the replica is only registered on the gateway/proxy after all its probes are ready (`is_job_ready`, jobs_running.py:1119-1130; `is_probe_ready` = success_streak >= spec.ready_after). If `model:` is set and probes omitted, a default /v1/chat/completions probe is added. For endpoint health-checking, either rely on probes+RUNNING or poll the service URL directly. -9) Do NOT set RunModel.status directly — use `switch_run_status()` (models.py:437 docstring) so an event is emitted; run termination is done by setting status TERMINATING + termination_reason (+ `skip_min_processing_interval=True`) and letting RunPipeline finish it — `stop_runs` does exactly this. -10) Background processing can be disabled per-replica (`SERVER_BACKGROUND_PROCESSING_ENABLED`, app.py:178); pipelines poll the DB, and the optional `pipeline_hinter` (services/pipelines.py, a FastAPI dependency) only reduces latency — passing None from a background task is fine, or use `get_pipeline_manager().hinter` in-process. -11) `submit_run` deletes any existing finished run with the same run_name (runs/__init__.py:716) — reusing a deterministic endpoint-derived run name will silently erase previous run history. -12) Runs with `schedule` in the profile start as PENDING with 0 replicas (runs/__init__.py:730-732); otherwise initial status SUBMITTED. diff --git a/endpoint-implementation-research/service-runtime-health.md b/endpoint-implementation-research/service-runtime-health.md deleted file mode 100644 index 76c69df5db..0000000000 --- a/endpoint-implementation-research/service-runtime-health.md +++ /dev/null @@ -1,144 +0,0 @@ -# service-runtime-health - -## Summary -Services with `model:` set get an OpenAI-compatible endpoint via a ServiceSpec stored on RunModel; without a gateway, the in-server proxy serves `/proxy/services///...` and `/proxy/models//...` (the latter marked deprecated but functional). Auth is a Bearer token of any project member, but server code never needs HTTP: probes already health-check model endpoints in-process by SSH-tunneling to the replica's service port (`get_service_replica_client(job)` in `job_replica_http_client.py`) and POSTing `{prefix}/chat/completions`. A default chat-completions probe is auto-generated for OpenAI-format models, executed every 3s by the multi-replica-safe `process_probes` scheduled task, and probe success (`success_streak >= ready_after`) gates the `JobModel.registered` flag which is what "replica ready" means. The endpoint health-checker should reuse the exact probe mechanism (job-level SSH tunnel + httpx over UDS), which works identically for gateway and non-gateway services. - -## Key files -- src/dstack/_internal/server/background/scheduled_tasks/probes.py — process_probes, _process_probe_async, _execute_probe — THE reference implementation for health-checking a deployed model: batch-locks due ProbeModels (multi-replica safe), tunnels to the replica, POSTs the probe request, updates success_streak. `_execute_probe` (lines 106-126) is exactly what the endpoint health-checker should copy. -- src/dstack/_internal/server/services/jobs/job_replica_http_client.py — get_service_replica_client(job: JobModel) -> AsyncGenerator[httpx.AsyncClient, None] — Async context manager giving an httpx client to a job's service port over an SSH tunnel + Unix socket. No HTTP auth, no gateway needed. Lines 21-27. -- src/dstack/_internal/server/services/jobs/job_replica_tunnel.py — get_service_replica_tunnel, SSH_CONNECT_TIMEOUT — Builds the tunnel via container_ssh_tunnel(job, forwarded_sockets=[SocketPair(remote=IPSocket('localhost', job_spec.service_port), local=UnixSocket(...))]). -- src/dstack/_internal/server/services/jobs/configurators/base.py — _probes (line 410), _probe_config_to_spec (458), _openai_model_probe_spec (472) — Default probe generation: if ServiceConfiguration.probes is None and model is OpenAIChatModel, generates POST {prefix}/chat/completions probe with body {model, messages:[{role:user,content:'hi'}], max_tokens:1}, timeout 30s. -- src/dstack/_internal/server/services/services/__init__.py — register_service (49), _register_service_in_gateway (99), _register_service_in_server (240), _get_service_spec (320) — Where `model:` becomes URLs. In-server: service_url=/proxy/services/{project}/{run}/ (273), model_url=service_url+prefix for openai / /proxy/models/{project}/ for tgi (274-277). Gateway: https://{run_name}.{wildcard_domain}, model at gateway.{wildcard_domain} (160-172). Result stored as run_model.service_spec JSON (96). -- src/dstack/_internal/server/services/proxy/repo.py — ServerProxyRepo.get_service (47), list_models (143), get_model (172) — In-server proxy data source. get_service only returns runs with gateway_id IS NULL, JobStatus.RUNNING, registered==True, job_num==0. list_models requires RunStatus.RUNNING + service_spec.options['openai']['model']. -- src/dstack/_internal/proxy/lib/services/service_connection.py — ServiceConnectionPool, ServiceConnection, get_service_replica_client(service, repo, service_conn_pool) (140) — Proxy-lib variant of replica client (pooled, persistent tunnels). Used by the in-server proxy and model proxy routers. -- src/dstack/_internal/proxy/lib/services/model_proxy/clients/openai.py — OpenAIChatCompletions.generate/stream — Existing typed client that POSTs {prefix}/chat/completions given an httpx.AsyncClient — reusable for a higher-level health check with response validation. -- src/dstack/_internal/proxy/lib/routers/model_proxy.py — get_models, post_chat_completions — OpenAI-compatible HTTP API: GET /proxy/models/{project}/models, POST /proxy/models/{project}/chat/completions. Mounted in app.py:261 with deprecated=True (still functional). -- src/dstack/_internal/server/services/probes.py — probe_model_to_probe, is_probe_ready — is_probe_ready(probe, spec) = probe.success_streak >= spec.ready_after. -- src/dstack/_internal/server/services/runs/__init__.py — is_job_ready (1226), submit_run (register_service call at 760), run_model_to_run (service_spec at 979-1004) — is_job_ready = all probes ready; Run.service: Optional[ServiceSpec] populated from RunModel.service_spec. -- src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py — _initialize_running_job_probes (1101), _maybe_register_replica (1119), _register_service_replica (1162) — Probes created when job reaches RUNNING; JobModel.registered set True only when all probes ready (and gateway registration succeeds if gateway-based). -- src/dstack/_internal/core/models/configurations.py — ProbeConfig (365), ServiceConfiguration.model (993), .probes (1019), convert_model validator (1058), probe constants (61-71) — model: str is coerced to OpenAIChatModel(format='openai'); OPENAI_MODEL_PROBE_TIMEOUT=30, DEFAULT_PROBE_INTERVAL=15, DEFAULT_PROBE_READY_AFTER=1. -- src/dstack/_internal/core/models/runs.py — ProbeSpec (245), JobSpec.probes/service_port (299-301), Probe (390), JobSubmission.probes (428), ServiceModelSpec (635), ServiceSpec (643), RunStatus (652) — ProbeSpec is the resolved probe stored in JobSpec; Probe(success_streak) surfaces per-submission probe state to clients. -- src/dstack/_internal/proxy/lib/deps.py — ProxyAuth, ProxyAuthContext — Bearer-token auth for proxy routes; enforced iff service.auth (service proxy) and always for model proxy. -- src/dstack/_internal/proxy/lib/schemas/model_proxy.py — ChatCompletionsRequest, ChatCompletionsResponse, ChatCompletionsChunk, ModelsResponse — Typed OpenAI-compatible request/response schemas already in the codebase. - -## Details -## 1. The `model:` property on services - -`ServiceConfiguration.model: Optional[AnyModel]` — `src/dstack/_internal/core/models/configurations.py:993-1003`. A plain string is coerced by the `convert_model` validator (configurations.py:1058-1062): - -```python -@validator("model", pre=True) -def convert_model(cls, v: Optional[Union[AnyModel, str]]) -> Optional[AnyModel]: - if isinstance(v, str): - return OpenAIChatModel(type="chat", name=v, format="openai") - return v -``` - -Model types (`src/dstack/_internal/core/models/services.py`): `BaseChatModel` (13), `TGIChatModel` (21), `OpenAIChatModel` (58, `prefix` default `"/v1"` at line 72), `ChatModel = Annotated[Union[TGIChatModel, OpenAIChatModel], Field(discriminator="format")]` (75), `AnyModel = Union[ChatModel]` (76). - -**Registration flow**: `submit_run` calls `await services.register_service(session, run_model, run_spec)` at `src/dstack/_internal/server/services/runs/__init__.py:760` (comment: "FIXME: Register services asynchronously in the background"). `register_service` (`src/dstack/_internal/server/services/services/__init__.py:49-96`) resolves the gateway (explicit ref / default / `gateway: false`) and either: -- `_register_service_in_gateway` (line 99): registers on every gateway connection via `client.register_service(...)` (`src/dstack/_internal/server/services/gateways/client.py:37-74`; when `"openai" in options` it also calls `register_openai_entrypoint` for `gateway.`, lines 52-54). -- `_register_service_in_server` (line 240): no gateway. Rejects SGLang router, non-auto https, autoscaling (min!=max), rate_limits. Then builds URLs (lines 273-277): - -```python -service_url = f"/proxy/services/{run_model.project.name}/{run_model.run_name}/" -if isinstance(run_spec.configuration.model, OpenAIChatModel): - model_url = service_url.rstrip("/") + run_spec.configuration.model.prefix -else: - model_url = f"/proxy/models/{run_model.project.name}/" -``` - -`_get_service_spec` (line 320-331) produces `ServiceSpec(url=service_url)` with `model=ServiceModelSpec(name, base_url=model_url, type)` and `options=get_service_options(configuration)`. `get_service_options` (`src/dstack/_internal/server/services/services/options.py:48-53`) sets `options["openai"] = {"model": conf.model.dict()}` after `complete_service_model` (options.py:10-23) fills TGI `chat_template`/`eos_token` from HuggingFace (network call, may raise `ServerClientError`). Result stored as `run_model.service_spec = service_spec.json()` (services/__init__.py:96). Gateway registration failure raises (`GatewayError` / `ServerClientError`) synchronously from submit. - -`ServiceModelSpec` / `ServiceSpec` — `src/dstack/_internal/core/models/runs.py:635-649`; `Run.service: Optional[ServiceSpec]` (runs.py:693) is populated in `run_model_to_run` (`server/services/runs/__init__.py:979-1004`). - -**In-server model listing**: `ServerProxyRepo.list_models` (`src/dstack/_internal/server/services/proxy/repo.py:143-170`) — runs where `gateway_id IS NULL`, `service_spec IS NOT NULL`, `RunStatus.RUNNING`, and `service_spec.options["openai"]["model"]` present. `get_model` (172-178) picks the newest by `created_at` on name collision. - -## 2. URL formats - -**Without a gateway (in-server proxy, always available unless `settings.FORBID_SERVICES_WITHOUT_GATEWAY`, services/__init__.py:89-95):** -- Service: `/proxy/services/{project_name}/{run_name}/{path}` — router `src/dstack/_internal/server/services/proxy/routers/service_proxy.py:31-49`, mounted at `src/dstack/_internal/server/app.py:260`. -- Model (OpenAI-compatible): `GET /proxy/models/{project_name}/models` and `POST /proxy/models/{project_name}/chat/completions` — router `src/dstack/_internal/proxy/lib/routers/model_proxy.py:26-65`, mounted at app.py:261 **with `deprecated=True`** (still functional; for `format: openai` models the ServiceSpec now points clients at `/proxy/services//` instead, i.e. straight through the service proxy — the model proxy is mainly needed for TGI-format translation). -- These are paths relative to the dstack server's base URL; auth via Bearer token (see below). - -**With a gateway** (services/__init__.py:160-177): service at `{http|https}://{run_name}.{wildcard_domain}`; model endpoint at `{http|https}://gateway.{wildcard_domain}` (or `service_url + model.prefix` for openai format). Requires wildcard domain DNS; served by Nginx on the gateway instance. - -## 3. Auth for calling through the proxy - -- Both proxy routers use `ProxyAuth` (`src/dstack/_internal/proxy/lib/deps.py:87-106`): `HTTPBearer` credentials → `ProxyAuthContext.enforce()` (deps.py:71-84) → `BaseProxyAuthProvider.is_project_member(project_name, token)`. Server implementation: `ServerProxyAuthProvider` (`src/dstack/_internal/server/services/proxy/auth.py:7-12`) → `is_project_member(session, project_name, token)` (`src/dstack/_internal/server/security/permissions.py:278-283`). So the token is **any project member's user token** (the same `Authorization: Bearer ` used for the REST API). -- Model proxy always enforces (`APIRouter(dependencies=[Depends(ProxyAuth(auto_enforce=True))])`, model_proxy.py:23). Service proxy enforces only when `service.auth` is true (`server/services/proxy/services/service_proxy.py:39-40`; `ServiceConfiguration.auth` default `True`, configurations.py:1012). -- **In-process alternative: yes.** Server code does not need HTTP+token at all — see mechanism below. There is no in-process function that "calls the FastAPI route"; instead server code opens the same SSH tunnel the proxy uses. - -## 4. Service probes - -- Config: `ProbeConfig` (`configurations.py:365-459`): `type: Literal["http"]`, `url` (default `/`), `method` (default `get`), `headers`, `body`, `timeout` (default 10s, min 1s), `interval` (default 15s), `ready_after` (default 1), `until_ready` (default False). `ServiceConfiguration.probes: Optional[list[ProbeConfig]]` (configurations.py:1019-1026) — `None` means "default"; docstring: "If `model` is set, defaults to a `/v1/chat/completions` probe." -- Default generation: job configurator `_probes()` (`server/services/jobs/configurators/base.py:410-419`) — explicit probes are converted by `_probe_config_to_spec` (458); otherwise, if `isinstance(model, OpenAIChatModel)`, returns `[_openai_model_probe_spec(model.name, model.prefix)]` (472-491): `POST {prefix}/chat/completions`, JSON body `{"model": , "messages": [{"role": "user", "content": "hi"}], "max_tokens": 1}`, `Content-Type: application/json`, `timeout=OPENAI_MODEL_PROBE_TIMEOUT` (30s, configurations.py:71), `interval=15`, `ready_after=1`. Note: **TGI-format models get no default probe.** Resolved probes live in `JobSpec.probes: list[ProbeSpec]` (`core/models/runs.py:301`, spec class at 245-255). -- DB: `ProbeModel` (`server/models.py:1117-1133`): `id, name, job_id, probe_num, due, success_streak, active`; unique `(job_id, probe_num)`. -- Lifecycle: created when the job first reaches RUNNING — `_initialize_running_job_probes` (`server/background/pipeline_tasks/jobs_running.py:1101-1116`). Evaluated by scheduled task `process_probes` (`server/background/scheduled_tasks/probes.py:29-79`) registered at `IntervalTrigger(seconds=3, jitter=1)` (`server/background/scheduled_tasks/__init__.py:47`). Multi-replica-safe pattern: `get_locker(get_db().dialect_name).get_lockset(ProbeModel.__tablename__)` + `select(...).where(due <= now, active == True).limit(100).with_for_update(skip_locked=True, key_share=True)`; actual HTTP executed off-session via an `AsyncIOScheduler` job (`_process_probe_async`, 82-103) which then updates `success_streak` (reset to 0 on failure, +1 on success) and `due = now + interval`. -- Execution (`_execute_probe`, probes.py:106-126): - -```python -async with get_service_replica_client(probe.job) as client: - resp = await client.request( - method=probe_spec.method, - url="http://dstack" + probe_spec.url, # host is dummy; transport is a UDS - headers=[(h.name, h.value) for h in probe_spec.headers], - content=probe_spec.body, - timeout=probe_spec.timeout, - follow_redirects=False, - ) - return resp.is_success -``` - -- Status reflection: `Probe(success_streak=int)` (`core/models/runs.py:390-391`) in `JobSubmission.probes` (runs.py:428) via `probe_model_to_probe` (`server/services/probes.py:5-6`). Readiness: `is_probe_ready(probe, spec) = probe.success_streak >= spec.ready_after` (`server/services/probes.py:9-10`); `is_job_ready(probes, probe_specs)` = all ready (`server/services/runs/__init__.py:1226-1227`). Probe failures do NOT fail/terminate the job; they only delay/gate readiness. Probes deactivate when the job leaves RUNNING or when `until_ready` and threshold reached (probes.py:63-69). - -## 5. Service readiness today - -- Job statuses: SUBMITTED → PROVISIONING → PULLING → RUNNING → ... (`core/models/runs.py:62-78`); run statuses PENDING/SUBMITTED/PROVISIONING/RUNNING/TERMINATING/... (runs.py:652-667). -- Run status aggregation: `_analyze_active_run` + `_get_active_run_transition` (`server/background/pipeline_tasks/runs/active.py:182-236, 369-409`): the run is RUNNING as soon as **any** replica job is RUNNING (active.py:393-394). So `RunStatus.RUNNING` alone does NOT mean the model answers requests. -- The real readiness signal is `JobModel.registered` (`server/models.py:578`, "whether the replica is registered to receive service requests"): `_maybe_register_replica` (`jobs_running.py:1119-1159`) sets `registered=True` only for services when `job_num == 0`, probes exist and `is_job_ready(...)` is True; for gateway services it first registers the replica on the gateway (`_register_service_replica`, 1162+; in-server case returns None and just flips the flag). The in-server proxy only routes to `registered == True` jobs (`ServerProxyRepo.get_service`, repo.py:57). -- Rolling deployment: driven by `deployment_num` comparison (`_has_out_of_date_replicas`, active.py:627-642) with surge/teardown in `_build_rolling_deployment_maps` (active.py:645-701); new replicas must become `registered` (probe-ready) before old ones are torn down. `ready_after`/`until_ready` on probes exist specifically for this. - -## 6. Concrete mechanism for the endpoint health-checker - -**Recommended (verified, no new plumbing): replicate `_execute_probe`.** Given the deployed service's RunModel, pick a `JobModel` with `status == JobStatus.RUNNING` (and `job_num == 0`), then: - -```python -from dstack._internal.server.services.jobs.job_replica_http_client import get_service_replica_client -import orjson - -body = orjson.dumps({ - "model": model_name, - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 1, -}).decode() -async with get_service_replica_client(job_model) as client: # SSH tunnel + httpx over UDS - resp = await client.request( - method="post", - url="http://dstack/v1/chat/completions", # or prefix.rstrip('/') + '/chat/completions' - headers=[("Content-Type", "application/json")], - content=body, - timeout=30, - ) - ok = resp.is_success -``` - -This is exactly what probes do today (probes.py:106-126) and works identically for gateway and non-gateway services because it tunnels straight to the replica (bypasses gateway/proxy/auth). Requires the job to have `job_spec.service_port` set (true for all services since 0.19.19; `get_service_port` fallback at `core/models/runs.py:757-762`). `get_service_replica_client(job)` signature: `server/services/jobs/job_replica_http_client.py:22-27`; underlying tunnel `get_service_replica_tunnel` uses `container_ssh_tunnel(job, forwarded_sockets=..., options=...)` (`server/services/ssh.py:81-98`), `SSH_CONNECT_TIMEOUT = timedelta(seconds=10)` (`job_replica_tunnel.py:20`). - -**Even simpler for the endpoint feature**: don't run your own HTTP check at all — rely on the existing probe machinery. If the authored service config has `model:` set (OpenAI format) and `probes` unset, dstack auto-creates the chat-completions probe; the endpoint processor can then just poll `JobModel.registered` (or `is_job_ready(job_model.probes, job_spec.probes)` via `server/services/runs/__init__.py:1226`) to decide active/failed. `run.status == RunStatus.RUNNING` + `job.registered == True` == "model verified to answer /v1/chat/completions". - -**Higher-level typed alternative** (validates the OpenAI response shape): use the proxy-lib stack in-process — `ServerProxyRepo(session).get_model(project, model_name)` + `.get_service(project, run_name)` (`server/services/proxy/repo.py:47,172`), `get_service_replica_client(service, repo, service_conn_pool)` (`proxy/lib/services/service_connection.py:140-163`, pool from `app.state.proxy_dependency_injector` — `ServerProxyDependencyInjector` set in app.py:106, defined in `server/services/proxy/deps.py:11`), then `get_chat_client(model, http_client).generate(ChatCompletionsRequest(model=..., messages=[...], max_tokens=1))` (`proxy/lib/services/model_proxy/model_proxy.py:11`, `clients/openai.py:21-33`, request schema `proxy/lib/schemas/model_proxy.py:11-27`). Caveat: `ServerProxyRepo.get_service` only returns non-gateway, `registered` services — so this path can't health-check gateway-based or not-yet-registered replicas. - -**HTTP path (not recommended for server-internal use)**: `POST {server}/proxy/models/{project}/chat/completions` with `Authorization: Bearer ` — requires a valid project-member token and is marked deprecated in app.py:261. - -## Gotchas -1. Run `RunStatus.RUNNING` fires when ANY replica job is RUNNING (runs/active.py:393-394) — it is NOT "model ready". The readiness signal is `JobModel.registered` (set only after all probes pass, jobs_running.py:1119-1130). An endpoint health-checker that only watches run status would mark endpoints active before the model can serve. -2. `/proxy/models/...` router is mounted with `deprecated=True` (app.py:261) — still works, but new code/URLs should prefer the service-proxy path `/proxy/services///chat/completions` for openai-format models, which is what `_register_service_in_server` now advertises (services/__init__.py:275). -3. The default chat-completions probe is generated ONLY when `model` is `OpenAIChatModel` and `probes` is None (`_probes()`, configurators/base.py:410-419). A bare string `model: ` coerces to OpenAIChatModel, so the common case is covered; TGI-format models get NO default probe. -4. Probes never fail the job — a permanently unhealthy service stays RUNNING with `registered=False` forever. An endpoint feature that wants a "failed" terminal state must add its own timeout on top. -5. `get_service_replica_client` exists TWICE with different signatures: `server/services/jobs/job_replica_http_client.py:22` takes a `JobModel` (context manager, fresh tunnel per call — use this from background tasks) vs `proxy/lib/services/service_connection.py:140` takes `(Service, BaseProxyRepo, ServiceConnectionPool)` (pooled, used by proxy routers). Don't confuse them in the plan. -6. `ServerProxyRepo.get_service`/`list_models` exclude gateway-based runs (`RunModel.gateway_id.is_(None)`) and unregistered jobs — the in-server proxy cannot reach gateway services. The job-tunnel mechanism (probes path) works for both. -7. In probe execution the URL host is literally `"http://dstack"` — a placeholder; transport is a Unix socket. Don't "fix" it. -8. Gateway registration happens synchronously inside `submit_run` (runs/__init__.py:758-760, marked FIXME) and can raise; `complete_service_model` for TGI does a blocking HTTP call to huggingface.co (options.py:25-45). -9. Multi-replica-safe background pattern to copy: `process_probes` (scheduled_tasks/probes.py:29-79) uses `get_locker(get_db().dialect_name).get_lockset(...)` + `with_for_update(skip_locked=True, key_share=True)` + due-based scheduling; scheduled via `AsyncIOScheduler` in `background/scheduled_tasks/__init__.py` (there is also `background/pipeline_tasks/` for the pipeline-style processors). -10. Auth token for HTTP proxy calls is a plain dstack user token of any project member (permissions.py:278-283); there is no separate service/model API key concept. diff --git a/endpoint-implementation-research/settings-migrations-testing.md b/endpoint-implementation-research/settings-migrations-testing.md deleted file mode 100644 index 45fde85fb2..0000000000 --- a/endpoint-implementation-research/settings-migrations-testing.md +++ /dev/null @@ -1,127 +0,0 @@ -# settings-migrations-testing - -## Summary -Established ground truth for server settings, migrations, models, testing, linting, and packaging conventions in /Users/dstack/dstack. Settings are plain module-level constants in src/dstack/_internal/server/settings.py read via os.getenv/environ helpers with DSTACK_ or DSTACK_SERVER_ prefixes, documented in mkdocs/docs/reference/env.md; feature flags are DSTACK_FF_* on the FeatureFlags class in src/dstack/_internal/settings.py. Migrations are Alembic autogenerate (run from src/dstack/_internal/server with `alembic revision -m "..." --autogenerate`), rendered with render_as_batch=True for SQLite compat, stored under migrations/versions//. Models use DeclarativeBase BaseModel with UUIDType(binary=False) PKs, NaiveDateTime, EnumAsString, EncryptedString custom types, and pydantic-JSON-as-Text columns; multi-replica pipeline rows use PipelineModelMixin (lock_expires_at/lock_token/lock_owner). Test factories live in src/dstack/_internal/server/testing/common.py (not factories.py) with test_db/session fixtures in testing/conf.py; tests run via `uv run pytest src/tests` (add --runpostgres for Postgres). Lint is ruff 0.12.7, types are pyright standard mode; heavy deps are optional-dependencies extras (server, gateway, aws, ...) guarded by try/except ImportError. - -## Key files -- src/dstack/_internal/server/settings.py — module-level constants; JobNetworkMode enum — All server env-var settings; docstring says 'Documented in reference/env.md' -- src/dstack/_internal/settings.py — FeatureFlags, DSTACK_VERSION, DSTACK_RELEASE — Global (non-server) settings; FeatureFlags class holds DSTACK_FF_* flags -- src/dstack/_internal/utils/env.py — Environ.get/get_bool/get_int/get_enum/get_callback; environ singleton — Typed env-var parsing helper used by newer settings -- src/dstack/_internal/server/models.py — BaseModel, PipelineModelMixin, NaiveDateTime, EncryptedString, DecryptedString, EnumAsString, constraint_naming_convention, RunModel, VolumeModel — All ORM models and custom column types -- src/dstack/_internal/server/alembic.ini — — script_location=migrations; file_template puts revisions in /MM_DD_HHMM__.py -- src/dstack/_internal/server/migrations/env.py — — render_as_batch=True (line 82), compare_type=True (line 81), target_metadata=BaseModel.metadata (line 16) -- src/dstack/_internal/server/migrations/versions/2026/03_04_2221_5e8c7a9202bc_add_exports.py — — Recent example migration creating tables with UUIDType PKs, named FKs, batch_alter_table for indexes -- contributing/MIGRATIONS.md — — Alembic command, expand-and-contract rules, one-table-per-migration, CREATE INDEX CONCURRENTLY guidance -- contributing/DEVELOPMENT.md — — uv sync --all-extras; pre-commit; pyright standard mode -- CONTRIBUTING.md — — uv run ruff check --fix; uv run ruff format; uv run pytest src/tests [--runpostgres] -- src/dstack/_internal/server/testing/common.py — create_user, create_project, create_repo, create_run, create_job, create_fleet, create_volume, create_gateway, create_instance, get_run_spec, ComputeMockSpec — THE factory module (testing/factories.py does not exist) -- src/dstack/_internal/server/testing/conf.py — test_db, session, postgres_container fixtures — test_db parametrized sqlite/postgres via indirect=True; postgres uses testcontainers -- src/tests/conftest.py — pytest_addoption(--runpostgres/--runui), disable_feature_flags — Re-exports testing.conf fixtures; autouse fixture forces all FeatureFlags to False -- src/tests/_internal/server/conftest.py — client, test_log_storage, image_config_mock, mock_gateway_connection — Server-level fixtures incl. httpx ASGI client against dstack._internal.server.main:app -- src/tests/_internal/server/background/pipeline_tasks/test_volumes.py — — Canonical pipeline-task test: Fetcher/Worker fixtures, parametrized test_db, factories -- src/tests/_internal/server/background/scheduled_tasks/test_probes.py — — Canonical scheduled-task test: calls process_probes() directly with mocked deps -- pyproject.toml — — ruff/pyright/pytest config, dependency-groups dev, [project.optional-dependencies] extras incl. server/gateway/all -- src/dstack/_internal/server/services/storage/s3.py — — Canonical optional-dependency import pattern (BOTO_AVAILABLE flag + try/except ImportError) -- mkdocs/docs/reference/env.md — — User-facing docs for DSTACK_SERVER_* env vars (server section around lines 94-140) - -## Details -# Conventions cheat-sheet (verified against code, 2026-07-03) - -## 1. Server settings — `src/dstack/_internal/server/settings.py` - -Plain module-level constants evaluated at import time. Module docstring (settings.py:1-3): *"Environment variables read by the dstack server. Documented in reference/env.md"* — the doc file is `mkdocs/docs/reference/env.md` (server env vars listed ~lines 94-140 with `{ #ANCHOR }` markers). - -Naming: env var prefix is `DSTACK_SERVER_*` for server-behavior settings, bare `DSTACK_*` for cross-cutting ones (`DSTACK_DATABASE_URL`, `DSTACK_SENTRY_DSN`, `DSTACK_DB_POOL_SIZE`, `DSTACK_ACME_SERVER`). Python constant name drops the `DSTACK_` prefix (e.g. `SERVER_PORT`, `DATABASE_URL`). - -Declaration patterns (all real examples): -- String w/ default: `SERVER_HOST = os.getenv("DSTACK_SERVER_HOST", "localhost")` (settings.py:28) -- Int: `SERVER_PORT = int(os.getenv("DSTACK_SERVER_PORT", "8000"))` (settings.py:29); `MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25))` (settings.py:54) -- Presence-based bool flag (set-to-anything = true): `SERVER_BACKGROUND_PROCESSING_DISABLED = os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None` followed by `SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED` (settings.py:47-50). Same DISABLED/ENABLED pair pattern for `SERVER_CONFIG_DISABLED/ENABLED` (58-59), `DEFAULT_CREDS_DISABLED/ENABLED` (122-123), `SERVER_SSH_POOL_DISABLED/ENABLED` (152-153). -- Typed helper (`dstack._internal.utils.env.environ`, an `Environ` wrapper over `os.environ` — env.py:12-121, methods `get(name, *, default)`, `get_bool`, `get_int`, `get_enum(name, enum_cls, *, value_type, default)`, `get_callback(name, callback, *, default)`): e.g. `SERVER_METRICS_RUNNING_TTL_SECONDS = environ.get_int("DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS", default=3600)` (settings.py:83-85); enum example `JOB_NETWORK_MODE = environ.get_enum("DSTACK_SERVER_JOB_NETWORK_MODE", JobNetworkMode, value_type=int, default=DEFAULT_JOB_NETWORK_MODE)` (settings.py:178-183). -- Optional str normalized to None: `SERVER_DEFAULT_DOCKER_REGISTRY = os.getenv("DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY") or None` (settings.py:132). -- Dev-only settings live at bottom under a `# Development settings` comment (settings.py:156-165), e.g. `SQL_ECHO_ENABLED`, `SERVER_PROFILING_ENABLED`. - -Feature flags: NOT in server/settings.py. `class FeatureFlags` in `src/dstack/_internal/settings.py:40-52` — env vars of the form `DSTACK_FF_*`, class attrs must be bool, docstring says flags are temporary for large features in development. Current sole flag: `CLI_PRINT_JOB_CONNECTION_INFO = os.getenv("DSTACK_FF_CLI_PRINT_JOB_CONNECTION_INFO") is not None`. In tests, ALL feature flags are force-disabled by the autouse session fixture `disable_feature_flags` in `src/tests/conftest.py:51-62`; to test a flag, monkeypatch `FeatureFlags` per-test. - -## 2. Alembic migrations — `src/dstack/_internal/server/migrations/` - -Generate (contributing/MIGRATIONS.md:7-10): -```shell -cd src/dstack/_internal/server/ -alembic revision -m "" --autogenerate -``` -- `alembic.ini` lives at `src/dstack/_internal/server/alembic.ini`; `script_location = migrations`; `file_template = %%(year)d/%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s` and `recursive_version_locations = true` — so files land in `migrations/versions//MM_DD_HHMM__.py` (e.g. `versions/2026/06_19_0709_857d8fa7fcc5_add_gateway_replica_pipeline.py`). -- `migrations/env.py` uses `target_metadata = BaseModel.metadata` (env.py:16) and configures autogenerate with `compare_type=True, render_as_batch=True` (env.py:81-82). **`render_as_batch=True` is what makes ALTERs SQLite-compatible** — column adds/drops are emitted as `with op.batch_alter_table("
", schema=None) as batch_op:` blocks (see add_gateway_replica_pipeline migration lines 44-65). Plain `op.create_table`/`op.drop_table` used for new tables (add_exports migration lines 24-40). -- Column types in migrations: `sqlalchemy_utils.types.uuid.UUIDType(binary=False)` for UUIDs; `dstack._internal.server.models.NaiveDateTime()` for datetimes (migrations import `import dstack._internal.server.models` for this); `sa.String(length=100)`, `sa.Text()`, `sa.Boolean()`. -- Constraint naming comes from `constraint_naming_convention` (models.py:191-197): `pk_%(table_name)s`, `fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s`, `uq_%(table_name)s_%(column_0_name)s`, `ix_%(column_0_label)s` — migrations pass explicit names via `op.f(...)` (e.g. `name=op.f("fk_exports_project_id_projects")`). -- Multi-replica zero-downtime rules (contributing/MIGRATIONS.md): migrations must not break old replicas; use expand-and-contract for column removals/renames (remove = release 1: `deferred=True` stop reading; release 2: drop). Alter only ONE table per migration/transaction (Postgres ACCESS EXCLUSIVE deadlock risk, MIGRATIONS.md:47-49). Index creation should use `postgresql_concurrently=True` inside `with op.get_context().autocommit_block():`, pre-dropping with `if_exists=True` for retry safety (MIGRATIONS.md:51-75). -- Data backfills inside migrations use lightweight `sa.table(...)/sa.column(...)` partial definitions + `op.execute(sa.update(...))` — see 06_19_0709_857d8fa7fcc5 lines 22-89 (also shows the exact pattern used to retrofit pipeline columns: add nullable, backfill `last_processed_at=created_at` and status via `sa.case`, then `alter_column(..., nullable=False)`). -- `pyproject.toml` [tool.pyright] ignores `src/dstack/_internal/server/migrations/versions`. - -## 3. models.py conventions — `src/dstack/_internal/server/models.py` - -- Base: `class BaseModel(DeclarativeBase): metadata = MetaData(naming_convention=constraint_naming_convention)` (models.py:200-201). SQLAlchemy 2.0 style: `Mapped[...]` + `mapped_column(...)`. -- PK convention: `id: Mapped[uuid.UUID] = mapped_column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4)` (`UUIDType` from `sqlalchemy_utils`). Table names are plural snake_case via `__tablename__` ("runs", "volumes", "gateway_computes"). -- `PipelineModelMixin` (models.py:204-207) — REQUIRED for any row processed by the multi-replica-safe background pipeline machinery: `lock_expires_at: Mapped[Optional[datetime]] = mapped_column(NaiveDateTime)`, `lock_token: Mapped[Optional[uuid.UUID]] = mapped_column(UUIDType(binary=False))`, `lock_owner: Mapped[Optional[str]] = mapped_column(String(100))`. Models using it: RunModel, JobModel, GatewayModel, GatewayComputeModel, FleetModel, InstanceModel, VolumeModel, PlacementGroupModel, ComputeGroupModel, ServiceRouterWorkerSyncModel. Pipeline-processed models additionally carry `last_processed_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime)` (sometimes `skip_min_processing_interval: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())`) and a partial "fetch queue" index in `__table_args__`, e.g. VolumeModel (models.py:991-998): - ```python - Index("ix_volumes_pipeline_fetch_q", last_processed_at.asc(), - postgresql_where=deleted == false(), sqlite_where=deleted == false()) - ``` - RunModel's variant filters on status: `postgresql_where=status.not_in(RunStatus.finished_statuses())` (models.py:466-471). -- Custom column types (all `TypeDecorator`s defined in models.py): - - `NaiveDateTime` (models.py:57) — impl DateTime; strips tzinfo on write, re-attaches UTC on read. Used for every datetime column; python-side default is `dstack._internal.utils.common.get_current_datetime`. - - `EncryptedString` (models.py:107) — impl String; binds `DecryptedString` pydantic-dual model (models.py:83, has `plaintext`, `decrypted`, `exc`, `get_plaintext_or_error()`); encrypt/decrypt funcs injected via `EncryptedString.set_encrypt_decrypt(...)` (wired by importing `dstack._internal.server.services.encryption` for side effect — see src/tests/_internal/server/conftest.py:9). Used e.g. `UserModel.token: Mapped[DecryptedString] = mapped_column(EncryptedString(200), unique=True)` (models.py:218), `BackendModel.auth = EncryptedString(20000)`. - - `EnumAsString(enum_class, length, fallback_deserializer=None)` (models.py:152) — stores `enum.name` string; used for all status columns: `status: Mapped[VolumeStatus] = mapped_column(EnumAsString(VolumeStatus, 100), index=True)`. -- **No JSON column type.** Structured payloads are pydantic models serialized to `Text`: `run_spec: Mapped[str] = mapped_column(Text)` (models.py:445), `VolumeModel.configuration: Mapped[str] = mapped_column(Text)` (models.py:981), `JobModel.job_spec_data`, etc. Written as `run_spec.json()`, read as `RunSpec.parse_raw(run.run_spec)` (pydantic v1: `pydantic>=1.10.10,<2.0.0` + pydantic-duality). -- FKs: `mapped_column(ForeignKey("projects.id", ondelete="CASCADE"))` + separate `relationship()`. Soft-delete convention: `deleted: Mapped[bool]` (+ optional `deleted_at`, `to_be_deleted` for pipeline-driven deletion). Bools added later use `server_default=false()` (from `sqlalchemy.sql`). -- Docstrings placed under fields as bare strings explain semantics (e.g. models.py:437 `"""`status` must be changed only via `switch_run_status()`."""`). - -## 4. Testing - -- Factories are in `src/dstack/_internal/server/testing/common.py` (**`testing/factories.py` does not exist**; the package has `common.py`, `conf.py`, `matchers.py`). All async, take `session: AsyncSession` first, `session.add(...)` + `await session.commit()`, return the model. Exact names (with a few exact signatures): - - `async def create_user(session, name="test_user", created_at=..., global_role=GlobalRole.ADMIN, token=None, email=None, ssh_public_key=None, ssh_private_key=None, active=True, deleted=False) -> UserModel` (common.py:147) - - `async def create_project(session, owner=None, name="test_project", created_at=..., ssh_private_key="", ssh_public_key="", is_public=False, templates_repo=None, deleted=False) -> ProjectModel` (common.py:200) - - `async def create_repo(...)` (262), `async def create_backend(session, project_id, backend_type=BackendType.AWS, config=None, auth=None, ...)` (228) - - `def get_run_spec(repo_id, run_name="test-run", configuration_path="dstack.yaml", profile=..., configuration=None, ssh_key_pub="user_ssh_key", ...) -> RunSpec` (341) — default configuration is `DevEnvironmentConfiguration(ide="vscode")`; pass e.g. `ServiceConfiguration(...)` to override. - - `async def create_run(session, project, repo, user, fleet=None, gateway=None, run_name=None, status=RunStatus.SUBMITTED, ..., run_spec=None, ...) -> RunModel` (365) — serializes `run_spec.json()` into the Text column. - - `async def create_job(session, run, ...) -> JobModel` (422); `async def create_fleet(...)` (758), `get_fleet_spec` (792), `get_fleet_configuration` (806); `async def create_gateway(...)` (639), `create_gateway_compute` (683); `async def create_instance(...)` (850), `get_instance_offer_with_availability` (964); `async def create_volume(session, project, user, status=..., created_at=..., last_processed_at=..., deleted_at=..., ...)` (1069), `get_volume_configuration` (1150), `get_volume_provisioning_data` (1188); `create_compute_group` (573), `create_placement_group` (1206), `create_probe` (619), `create_secret` (1301), `list_events` (1317), `get_auth_headers(token)` (141). Also `class ComputeMockSpec` (common.py:1404) inheriting all Compute* capability mixins, "Can be used to create Compute mocks that pass all isinstance() asserts". -- DB fixtures — `src/dstack/_internal/server/testing/conf.py`, re-exported in `src/tests/conftest.py` and `src/tests/_internal/server/conftest.py`: - - `test_db` (pytest_asyncio fixture, function-scoped, conf.py:22-55): param "sqlite" (default; in-memory aiosqlite with StaticPool + check_same_thread=False, `BaseModel.metadata.create_all` — **migrations are NOT run in tests**) or "postgres" (testcontainers `PostgresContainer("postgres:16-alpine", driver="asyncpg")`, skipped unless `--runpostgres`; tables created once per session then `TRUNCATE ... RESTART IDENTITY CASCADE` between tests). Calls `override_db(db)` from `dstack._internal.server.db`. - - `session` (conf.py:58-62): `async with test_db.get_session() as session: yield session`. -- Background-task test pattern (pipeline): `src/tests/_internal/server/background/pipeline_tasks/test_volumes.py` — class-level decorators: - ```python - @pytest.mark.asyncio - @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) - class TestVolumeFetcher: - async def test_...(self, test_db, session: AsyncSession, fetcher: VolumeFetcher): ... - ``` - Fixtures construct pipeline components directly: `VolumeWorker(queue=Mock(), heartbeater=Mock(), pipeline_hinter=Mock())`, `VolumeFetcher(queue=asyncio.Queue(), queue_desired_minsize=1, min_processing_interval=timedelta(seconds=15), lock_timeout=timedelta(seconds=30), heartbeater=Mock())`; tests seed rows with factories, then `items = await fetcher.fetch(limit=10)` / call worker methods; lock behavior asserted via `lock_token/lock_expires_at/lock_owner`. Imports come from `dstack._internal.server.background.pipeline_tasks.volumes` (`VolumeFetcher, VolumePipeline, VolumePipelineItem, VolumeWorker`). Test dirs: `src/tests/_internal/server/background/pipeline_tasks/` (test_fleets.py, test_gateways.py, test_gateway_replicas.py, test_volumes.py, test_runs/, test_instances/, ...) and `.../scheduled_tasks/` (test_probes.py, test_events.py, ...). Scheduled-task tests call the task function directly (e.g. `from ...background.scheduled_tasks.probes import process_probes`; `pytestmark = pytest.mark.usefixtures("image_config_mock")`; freezegun `freeze_time` used). - Note: pytest-asyncio runs in default (strict) mode — every async test/class needs explicit `@pytest.mark.asyncio`; no `asyncio_mode` is set in pyproject. -- Other server fixtures (src/tests/_internal/server/conftest.py): `client` (httpx.AsyncClient over `ASGITransport(app=dstack._internal.server.main.app)`), `image_config_mock`, `test_log_storage`, `mock_gateway_connection`. Network is blocked by default: pytest addopts `--disable-socket --allow-hosts=127.0.0.1,localhost --allow-unix-socket` (pyproject.toml:117-122); pytest-env sets `DSTACK_SSHPROXY_API_TOKEN=test-token`, `DSTACK_CLI_RICH_FORCE_TERMINAL=0`. -- Running tests (CONTRIBUTING.md:49-64): `uv run pytest src/tests`; `uv run pytest src/tests --runpostgres` for Postgres. CI (build-artifacts.yml:93) runs `uv run pytest -n auto src/tests --runui $RUNPOSTGRES`. Custom options defined in src/tests/conftest.py: `--runui`, `--runpostgres`; markers `ui`, `postgres`, `windows`, `windows_only` (+ `shim_version`, `dockerized` in pyproject). - -## 5. Linting / type-checking - -- ruff (pyproject.toml:84-96): `target-version = "py310"`, `line-length = 99`, lint select `["E","F","I","Q","W","PGH","FLY","S113"]`, ignore `["E501","E712"]`, isort `known-first-party = ["dstack"]`. Pinned `ruff==0.12.7` in dev group ("should match .pre-commit-config.yaml"); pre-commit hooks `ruff` + `ruff-format` from astral-sh/ruff-pre-commit. Commands per CONTRIBUTING.md: `uv run ruff check --fix` then `uv run ruff format`. -- pyright (pyproject.toml:98-113): `typeCheckingMode = "standard"`; `include` is a whitelist — `src/dstack/plugins`, `src/dstack/_internal/server`, `src/dstack/_internal/core/services`, backends aws/kubernetes/runpod, cli configurators/commands, and `src/tests/_internal/server/background/pipeline_tasks`; ignores `src/dstack/_internal/server/migrations/versions`. CI runs `jakebailey/pyright-action@v3` after `uv sync --all-extras` (build-artifacts.yml:74-79). Local: `uv tool install pyright && pyright -p .` (DEVELOPMENT.md). No mypy. -- requires-python `>=3.10`; pydantic is v1 (`pydantic>=1.10.10,<2.0.0` + `pydantic-duality>=1.2.4`). - -## 6. Optional dependencies / extras (pyproject.toml:172-271) - -- Base `[project.dependencies]` is the lightweight CLI/client set (pyyaml, requests, paramiko, rich, pydantic v1, gpuhunt, apscheduler<4, ...). Build backend: hatchling. -- `[project.optional-dependencies]`: `gateway` (fastapi/starlette/uvicorn/httpx/jinja2/aiorwlock/aiocache), `server` (fastapi, starlette, uvicorn[standard], sqlalchemy[asyncio]>=2.0.0, sqlalchemy_utils, alembic>=1.16.0, aiosqlite, asyncpg, alembic-postgresql-enum, sentry-sdk[fastapi], prometheus-client, grpcio, protobuf, smg-grpc-proto==0.4.9, docker, python-dxf, httpx, jinja2, watchfiles, requests-unixsocket, python-json-logger), then per-backend extras that each depend on `dstack[server]`: `aws` (boto3/botocore), `azure`, `gcp`, `datacrunch`, `verda`, `kubernetes`, `lambda`, `oci`, `nebius`, `fluentbit` (fluent-logger + elasticsearch), `crusoe` (server only), and `all = dstack[gateway,server,aws,azure,gcp,verda,kubernetes,lambda,nebius,oci,crusoe,fluentbit]`. Dev tooling is in `[dependency-groups] dev` (pytest~=8.0, pytest-asyncio, pytest-xdist, freezegun, testcontainers, openai>=1.68.2 which is dev-only for gateway/OpenAI-compat tests, ruff, ...) — installed via `uv sync --all-extras` (installs extras + dev group). -- Runtime guard pattern for optional imports — module-level availability flag, class defined only when import succeeds (src/dstack/_internal/server/services/storage/s3.py:5-13): - ```python - BOTO_AVAILABLE = True - try: - import botocore.exceptions - from boto3 import Session - except ImportError: - BOTO_AVAILABLE = False - else: - class S3Storage(BaseStorage): ... - ``` - Same pattern in services/storage/gcs.py, services/logs/aws.py, services/logs/gcp.py, services/logs/fluentbit.py, services/plugins.py. An `anthropic` extra would follow this: new extra in [project.optional-dependencies] depending on `dstack[server]`, guarded import + availability flag, and inclusion in `all`. **No anthropic/openai runtime dependency currently exists in project dependencies or extras** (openai appears only in the dev group). - -## Gotchas -1) `src/dstack/_internal/server/testing/factories.py` DOES NOT EXIST — factories are in `testing/common.py`; importing "factories" would be a hallucination. 2) Tests create schema via `BaseModel.metadata.create_all`, NOT alembic — a new model works in tests without a migration, so a missing migration won't fail unit tests; the migration must still be authored for real deployments. 3) Migrations must be multi-replica/zero-downtime safe (expand-and-contract; only one table altered per migration; concurrent index creation with if_exists pre-drop) per contributing/MIGRATIONS.md — a naive autogenerated migration may violate this. 4) pytest runs with `--disable-socket` (only localhost allowed) — any Anthropic API client code in tests must be fully mocked, no network. 5) pytest-asyncio is in strict mode: every async test needs `@pytest.mark.asyncio`; Postgres coverage requires `@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)` and is skipped without `--runpostgres`. 6) pydantic is v1 (<2.0.0) with pydantic-duality — do not plan pydantic v2 APIs (`model_dump`, `field_validator`); persisted specs use `.json()`/`.parse_raw()`. 7) There is no JSON column type in models.py — structured data goes in `Text` columns as pydantic JSON. 8) Feature flags live in `dstack._internal.settings.FeatureFlags` (DSTACK_FF_*), not server/settings.py, and are auto-disabled in all tests by an autouse fixture — tests of a flag must monkeypatch FeatureFlags. 9) pyright only checks whitelisted paths (include list) — new dirs under `_internal/server` are covered automatically, but new test dirs are NOT unless added (only `src/tests/_internal/server/background/pipeline_tasks` is type-checked today). 10) `settings.py` env vars must be documented in `mkdocs/docs/reference/env.md` (settings.py module docstring says so; some defaults note 'keep in sync' with that doc). 11) Alembic autogenerate must be run from `src/dstack/_internal/server/` (alembic.ini lives there); revision files are placed in a year subdirectory automatically by file_template. 12) New pipeline-style tables need PipelineModelMixin columns + `last_processed_at` + a partial `ix_
_pipeline_fetch_q` index with BOTH `postgresql_where` and `sqlite_where`. 13) The `anthropic` package is not a dependency anywhere; `openai` exists only in the dev dependency group — an Anthropic SDK dep should be added as a new optional extra depending on dstack[server] with a try/except ImportError guard (S3Storage pattern). diff --git a/endpoint-implementation-research/skills-and-pr3856.md b/endpoint-implementation-research/skills-and-pr3856.md deleted file mode 100644 index 713be189bf..0000000000 --- a/endpoint-implementation-research/skills-and-pr3856.md +++ /dev/null @@ -1,85 +0,0 @@ -# skills-and-pr3856 - -## Summary -PR #3856 ("Add dstack-dev skill for using dev environments", author peterschmidt85) is OPEN and NOT merged (mergedAt=null, closedAt=null, no reviews, CI green; it was once stale-closed by a bot and reopened, last updated 2026-07-03). It adds a new skills/dstack-dev/SKILL.md (39 lines, a playbook for using dev environments + SSH to experiment/tune/debug workloads before promoting them to tasks/services) and appends 2 description lines to skills/dstack/SKILL.md. The local checkout has exactly one skill, skills/dstack/SKILL.md (567 lines), a Claude Code-format skill (YAML frontmatter name/description + markdown body, single file, no referenced files) teaching agents dstack YAML authoring for all 6 config types, exact CLI workflows (apply/-d/-y, ps, attach, offer, logs, stop), anti-hallucination rules, and service/model-endpoint details. Other agent-facing material: raw .md docs mirrored to the site (https://dstack.ai/docs/**.md), generated llms.txt (~5KB) and llms-full.txt (~482KB), inference example docs with ready-made service YAMLs (vllm/sglang/trtllm/nim/dynamo), and a published skill at https://dstack.ai/skill.md + /.well-known/skills/. Critically, NONE of this ships in the installed dstack wheel (skills/, mkdocs/, examples/ are all outside src/dstack; hatchling only packages src/dstack plus the gitignored statics via `artifacts`), and there are zero "anthropic"/"claude"/skill-loading references in src/dstack — a server-embedded agent needs its instructional context vendored into src/dstack/... (git-tracked files there ship automatically) or fetched over the network. - -## Key files -- skills/dstack/SKILL.md — frontmatter name=dstack; sections: Quick agent flow, Agent execution guidelines, Configuration types, Essential CLI commands, Troubleshooting — The only skill in the local checkout (567 lines). Claude Code skill format: YAML frontmatter (name: dstack; description: one-liner) + markdown body. Teaches: config types with YAML examples (dev-environment, task, service, fleet, volume, gateway), agent CLI workflows (echo "n" | dstack apply -f to preview; dstack apply -f -y -d to submit; dstack ps -v; background nohup dstack attach), anti-hallucination rules ('If a command or flag isn't documented, it doesn't exist'), service endpoints/auth/model base_url, troubleshooting, and links to raw-.md doc URLs plus https://dstack.ai/llms-full.txt. -- scripts/docs/hooks.py — SKILL_PATH=("skills","dstack","SKILL.md") (line 22), on_post_build (line 256), _write_well_known_skills (line 295), _generate_llms_files (line 342) — mkdocs hooks (registered in mkdocs.yml:19-20). on_post_build (line 256) copies every docs .md raw into the built site (so every page is fetchable as https://dstack.ai/docs/.md), _write_well_known_skills (line 295) publishes ONLY skills/dstack/SKILL.md (SKILL_PATH constant, line 22) to site root skill.md/SKILL.md and .well-known/skills/dstack/SKILL.md + index.json, _generate_llms_files (line 342) runs gen_llms_files.py. dstack-dev from PR #3856 would NOT be web-published without changing SKILL_PATH handling. -- scripts/docs/gen_llms_files.py — generate_llms_files(repo_root, site_dir, mkdocs_config) (line 224), INCLUDE_SECTIONS/EXCLUDE_SECTIONS (lines 15-16) — Generates llms.txt (nav-derived link index with descriptions) and llms-full.txt (raw concatenation of all included .md sources). INCLUDE_SECTIONS = ["Getting started", "Concepts", "Guides", "Examples"]; EXCLUDE_SECTIONS = ["Reference"] (line 15-16) — so the YAML schema reference pages are NOT in llms-full.txt. Local build artifacts: site/llms.txt (5,372 bytes), site/llms-full.txt (482,113 bytes); site/ is gitignored build output. -- pyproject.toml — [tool.hatch.build.targets.wheel] artifacts (line 65-68) — hatchling build backend (lines 50-52). Wheel/sdist only add artifacts "src/dstack/_internal/server/statics/**" (lines 60-68) beyond the auto-detected src/dstack package — `artifacts` exists solely because statics/ is gitignored (.gitignore line 26). skills/, mkdocs/, examples/ are repo-root dirs outside src/ and are NOT in the installed package. Git-tracked non-.py files inside src/dstack DO ship (e.g. nginx jinja templates, alembic.ini), so vendoring a committed .md under src/dstack/... ships automatically with no pyproject change. -- mkdocs/docs/installation.md — — Lines 208-222: 'Install agent skills' section — the documented consumption path is `npx skills add dstackai/dstack` via skills.sh (https://skills.sh/dstackai/dstack/dstack), for 'AI agents like Claude, Codex, and Cursor ... to help them use the CLI and edit configuration files'. -- mkdocs/docs/examples/inference/vllm.md — — Example of ready-made agent context for authoring service configs: full `type: service` YAML deploying Qwen/Qwen3.6-27B with vllm/vllm-openai:v0.19.1 on H100:4 (NVIDIA and AMD ROCm tabs), /root/.cache instance volume, model:/port: fields. Siblings: sglang.md, trtllm.md, nim.md, dynamo.md under mkdocs/docs/examples/inference/. All included in llms-full.txt (Examples section is in INCLUDE_SECTIONS). Served raw at https://dstack.ai/docs/examples/inference/vllm.md. -- AGENTS.md — — Repo-root contributor guidelines for agents working ON the dstack codebase (uv sync, uv run pytest, ruff, pyright, style rules). Not about using dstack to deploy; not packaged; not relevant as service-authoring context. -- mkdocs.yml — — site_url https://dstack.ai (line 3), docs_dir: mkdocs (line 17), hooks: scripts/docs/hooks.py (lines 19-20). Nav 'More' section links llms-full.txt and skill.md as external URLs (lines 397-398). - -## Details -## 1. PR #3856 — state and content - -`gh pr view 3856 --repo dstackai/dstack`: -- Title: "Add dstack-dev skill for using dev environments"; author `peterschmidt85` (Andrey Cheptsov); base `master`, head `skills-dstack-dev` (branch exists on origin, tip 7c2f5a034d10a541e4212b4599d9f5b13b04c915). -- **State: OPEN. mergedAt: null. closedAt: null. NOT merged — confirmed.** Created 2026-05-06, updated 2026-07-03. Bot comments show it was stale-marked and stale-closed once, then reopened. reviewDecision empty, zero reviews, mergeable=MERGEABLE, CI checks SUCCESS. -- Files: `skills/dstack-dev/SKILL.md` (+39 new) and `skills/dstack/SKILL.md` (+2). - -### skills/dstack-dev/SKILL.md content (from `gh pr diff 3856`) -Frontmatter: `name: dstack-dev`; description says it is "specifically designed for using dstack's dev environments when it's required to experiment with code, tune and debug workloads before running them as tasks or services". -Body — "Using dev environments for experimenting, tuning, and debugging code": -1. Enable the base `dstack` skill first (explicit cross-reference). -2. **Fleet**: use `dstack offer` to pick instance config (GPU count/name, disk sized for model weights); ensure/create a matching fleet; VM-based backends allow pre-provisioned `nodes: ` and instance reuse; container backends need `nodes: 0..` ranges; SSH fleets for on-prem; prefer VM-based when possible. -3. **Docker image**: pick the image you plan to use later in the task/service; mount optional instance volume at `/root/.cache` if the workload downloads models/kernels; `docker: true` (VM-based fleets only) to try multiple images via `docker run` in one run. -4. **Dev environment**: author dev-environment config pinned to the fleet, `dstack apply -d`. -5. **SSH**: once `running`, `dstack attach --logs`, then `ssh ` to run commands interactively; **both must run outside the sandbox or they fail**. -6. **Iterate**: stop run, change image/fleet, re-apply. -Then an "Example: inference" 10-step recipe: research framework/GPU/disk requirements from official docs/model card → `dstack offer` → fleet → image (+cache volume / `docker: true`) → dev env → attach → ssh + experiment → reuse the proven hardware+commands in a task or **service** configuration. - -This is directly the "agent prototypes on a dev environment, then promotes to a service" workflow. Because the PR is unmerged, `skills/dstack-dev/` **does not exist** in the local checkout (`ls /Users/dstack/dstack/skills/` shows only `dstack/`). - -### skills/dstack/SKILL.md change in the PR -Adds only 2 lines to the frontmatter description: "This skill covers dstack's core capabilities for managing fleets, dev environments, tasks, and services. It includes best practices for applying configurations, monitoring workloads, and troubleshooting common issues." The local checkout matches the pre-PR version (description is the single one-liner, `skills/dstack/SKILL.md:3-4`). - -## 2. Current skills/ directory (local checkout, master) - -`/Users/dstack/dstack/skills/` contains exactly one skill: `skills/dstack/SKILL.md` (567 lines). No other files, no `references/` subdir, no scripts. Detailed content: - -- **Frontmatter** (lines 1-5): `name: dstack`, multi-line `description` (one sentence). -- **Overview / How it works** (lines 9-31): server (local/remote/dstack Sky), CLI (project config in `~/.dstack/config.yml`, managed with `dstack project`), `*.dstack.yml` config files; `dstack apply` plan-then-submit semantics, attach-by-default, `-d` for detached. -- **Quick agent flow** (lines 33-39): `echo "n" | dstack apply -f ` to preview plan → `dstack apply -f -y -d` → `dstack ps -v` → attach in background if needed → escalate if sandboxed attach fails. -- **Anti-hallucination rules** (lines 41-52): "Never propose `dstack` CLI commands or YAML syntaxes that don't exist"; verify via `--help`; never invent flags; never guess YAML property names; never run `dstack apply` for runs without `-d` in automated contexts; don't reformat tabular CLI output; prefer `-y` over `echo "y" |`. -- **Agent execution guidelines** (lines 54-141): output accuracy, `--help` verification, blocking commands (`dstack attach`, `apply` without `-d`, `ps -w`), 10-60s timeouts otherwise, confirmation handling, detached-run follow-up, background attach pattern (`nohup dstack attach --logs > /tmp/.attach.log 2>&1 & echo $! > /tmp/.attach.pid`), sandbox-permission escalation guidance, "dstack run only supports `dstack run get --json`". -- **Configuration types** (lines 143-340): common params (repo/repos/files/image/docker:true/env/volumes/resources); best practices (always set `name`; env var *names* only with values in `.envrc`; `python` and `image` mutually exclusive); `files`/`repos` intent policy; then per-type YAML examples: dev-environment, task (ports, distributed `nodes`), **service** (vLLM example with `port: 8000`, `model: meta-llama/...`, `resources.gpu: 80GB, disk: 200GB`; endpoint URL schemes `/proxy/services///` and `https://./`; `Authorization: Bearer ` unless `auth: false`; model endpoint via `service.model.base_url` from `dstack run get --json`, OpenAI-compatible = `service.url` + `/v1`; curl chat/completions example), fleet (nodes ranges, `placement: cluster`, ssh_config), volume (network + instance-volume `/root/.cache` pattern with `optional: true`), gateway (when required: autoscaling, rate limits, HTTPS custom domain, WebSockets). -- **Essential CLI commands** (lines 342-503): apply workflows for run vs infra configs; `dstack fleet [get|delete -i]`; `dstack ps [-v|--json|-n]`; `dstack run get --json`; `dstack attach [--logs]`; `dstack logs [-d|--replica|--job]`; `dstack stop [-y|--abort]`; `dstack offer` with `--backend/--gpu/--fleet/--max-offers/--group-by gpu|backend|region|count/--json`. -- **Troubleshooting + resources** (lines 505-566): JSON-inspection recipes; common issues (No offers/No fleet/config errors/provisioning timeouts); links exclusively to raw-.md doc URLs (e.g. `https://dstack.ai/docs/concepts/services.md`, `https://dstack.ai/docs/reference/dstack.yml/service.md`, `https://dstack.ai/docs/guides/troubleshooting.md`) and "Full documentation: https://dstack.ai/llms-full.txt" (line 566). - -Git history of the file: last touched by #3869 (CLI & API guide rework) and #3859 (docs/ → mkdocs/ rename); substantive skill edits in #3774 (offer --fleet), #3634, #3560, #3555, #3554, #3547. - -## 3. How the skills are consumed - -- **Format**: standard agent-skill (Claude Code) format — YAML frontmatter with only `name` and `description`, markdown body. Single self-contained SKILL.md; `.well-known/skills/index.json` lists `"files": ["SKILL.md"]` — no referenced auxiliary files. -- **Install channel (documented)**: `npx skills add dstackai/dstack` via skills.sh — `mkdocs/docs/installation.md:208-222` ("Install agent skills", links https://skills.sh/dstackai/dstack/dstack), cross-referenced from `quickstart.md:9` and `guides/cli-api.md:9`. This reads the GitHub repo's `skills/` directory. -- **Web publication**: `scripts/docs/hooks.py` at docs build: `_write_well_known_skills` (hooks.py:295-339) parses `skills/dstack/SKILL.md` frontmatter and copies it to site root as both `skill.md` and `SKILL.md`, plus `.well-known/skills/dstack/SKILL.md` and `.well-known/skills/index.json`. Verified in local build output `site/.well-known/skills/index.json`. Note `SKILL_PATH = ("skills", "dstack", "SKILL.md")` (hooks.py:22) is hard-coded to the single dstack skill — merging PR #3856 would NOT auto-publish dstack-dev to the website (only to the git repo / skills.sh path). -- **This session**: a `dstack` skill is also loaded in the current agent environment (Skill tool list), with the same one-line description as the local SKILL.md frontmatter. - -## 4. Other agent-facing material a server-embedded agent could use - -- **llms.txt / llms-full.txt**: generated at docs build by `scripts/docs/gen_llms_files.py` (invoked from hooks.py:342-365). `llms.txt` = nav-driven link index (title + frontmatter description per page, URLs ending in `.md`). `llms-full.txt` = raw concatenation of all pages in sections Getting started, Concepts, Guides, Examples (`INCLUDE_SECTIONS`, gen_llms_files.py:15); the Reference section (dstack.yml schema reference, HTTP API) is EXCLUDED (line 16). Local build: `site/llms.txt` 5,372 B; `site/llms-full.txt` 482,113 B (~120k tokens — too big to inline whole into a prompt; would need slicing). Live at https://dstack.ai/llms.txt and https://dstack.ai/llms-full.txt. `site/` is gitignored build output — not a source artifact. -- **Raw markdown docs endpoints**: hooks.py `on_post_build` (256-292) copies every `mkdocs/docs/**/*.md` verbatim into the built site (with `#SCHEMA#` placeholders in `docs/reference/**` expanded via `gen_schema_reference.py`), and `mimetypes.add_type("text/plain", ".md")` (hooks.py:17). So every doc page, including the full YAML config reference `docs/reference/dstack.yml/service.md`, is fetchable as plain markdown from dstack.ai. -- **Inference examples with ready service YAMLs**: `mkdocs/docs/examples/inference/{vllm,sglang,trtllm,nim,dynamo}.md` — each contains complete `type: service` configs for a concrete model (vllm.md: Qwen/Qwen3.6-27B, image vllm/vllm-openai:v0.19.1, H100:4, NVIDIA + AMD tabs, cache instance volume). Index page `mkdocs/docs/examples.md`. Source-code examples live in repo `examples/` (distributed-training, llms, misc, models, plugins, server-deployment, single-node-training). These example doc pages ARE included in llms-full.txt. -- **AGENTS.md** (repo root): codebase-contributor guidelines (uv, pytest, ruff, style) — for agents developing dstack itself, not for authoring service configs. - -## 5. Installed package vs vendoring (key packaging facts) - -- Build backend is hatchling (pyproject.toml:50-52); the wheel auto-detects `src/dstack` and the only extra `artifacts` entry is `src/dstack/_internal/server/statics/**` (pyproject.toml:60-68) — needed because statics is gitignored (.gitignore:26) and hatchling excludes VCS-ignored files by default. -- **`skills/`, `mkdocs/`, `examples/`, `site/llms*.txt` are all repo-root paths outside `src/` and are NOT part of the installed `dstack` package.** A server-embedded agent cannot `importlib.resources`-load any of today's skill/doc material from a pip-installed dstack. -- Precedent for vendoring: git-tracked non-.py files inside `src/dstack` do ship in the wheel today (e.g. `src/dstack/_internal/proxy/gateway/resources/nginx/*.jinja2`, `src/dstack/_internal/core/backends/template/*.jinja`, `src/dstack/_internal/server/alembic.ini`). So committing e.g. `src/dstack/_internal/server/resources/agent/SKILL.md` (or prompt .md files) ships automatically with no pyproject change; only gitignored/generated files would need an `artifacts` entry. -- **No existing agent plumbing in the server**: `grep -ri anthropic src/dstack --include=*.py` → 0 hits; no "claude" hits; no code in `src/dstack/_internal/server` references skills or SKILL.md. The `openai` package appears only in the dev dependency group (pyproject.toml:149) for tests. Any Anthropic API client, prompt loading, or skill embedding is net-new work. -- Alternative to vendoring: fetch https://dstack.ai/skill.md or https://dstack.ai/llms-full.txt at runtime — but that adds a network/version-skew dependency, and note the CLI-driving portions of SKILL.md (attach/nohup/sandbox guidance) target an interactive human-machine CLI agent, not a server-side API-driven agent; a server-embedded agent would author configs via the Python API/HTTP API, so only the YAML-authoring sections (Configuration types, service endpoint semantics) and the inference example YAMLs are directly reusable context. - -## Gotchas -1. PR #3856 is OPEN, not merged — `skills/dstack-dev/` does not exist on master or in the local checkout; do not reference it as available. It also has zero reviews and was stale-closed once (reopened 2026-07-03), so do not assume imminent merge. -2. The skills are NOT in the installed Python package. Nothing under repo `skills/`, `mkdocs/`, or `examples/` ships in the dstack wheel (hatchling packages only src/dstack + gitignored statics via `artifacts`). A plan that says "server loads skills/dstack/SKILL.md from the package" is wrong — it must vendor content under src/dstack (git-tracked files there ship automatically) or fetch from dstack.ai. -3. There is zero existing Anthropic/agent/skill-loading code in src/dstack (grep confirms 0 hits for "anthropic"); an anthropic SDK dependency would also be net-new (not in any dependency group). -4. skills/dstack/SKILL.md is written for a CLI-driving interactive agent (sandbox escalation, nohup attach, `echo "n" | dstack apply`). A server-embedded agent uses the API, so only parts of it (config-type YAML sections, service endpoint/auth/model-URL semantics) transfer; the CLI-workflow sections would be misleading context. -5. llms-full.txt is ~482KB and EXCLUDES the Reference section — the authoritative dstack.yml property reference (docs/reference/dstack.yml/service.md) is only available as a separately-fetched raw .md URL (schema-expanded at build time), not inside llms-full.txt. -6. hooks.py hard-codes SKILL_PATH to the single dstack skill; the website publishing path (dstack.ai/skill.md, .well-known/skills) covers only that one skill even after #3856 merges. -7. site/ (containing built llms.txt/llms-full.txt/skill.md) is gitignored build output — these files exist only after a docs build; don't treat site/ paths as source artifacts. -8. Recent renames: docs/ was renamed to mkdocs/ and examples moved under mkdocs/docs/examples (PR #3859); older references to a top-level docs/ directory are stale. diff --git a/endpoint-implementation-research/verify-findings.md b/endpoint-implementation-research/verify-findings.md deleted file mode 100644 index a2de55e57a..0000000000 --- a/endpoint-implementation-research/verify-findings.md +++ /dev/null @@ -1,147 +0,0 @@ -# Verifier 0 - -OVERALL: The plan's sections 2-5 are exceptionally well-grounded: every file path, symbol name, and cited line number I checked (configurations.py:1316/1384/1393/1401/1440, models.py:204/951, volumes.py:259, events.py:12, services/events.py:89, jobs_running.py:1158, probes.py:106, plan.py:970, configurators/__init__.py:27) resolves exactly as claimed, and the mechanism descriptions (lock dance, pipeline fetch/lease, pydantic-duality rules, ProjectMember vs ProjectAdmin, alembic conventions) match the code verbatim. I found no blockers or majors — only four minor corrections: `registered` is in fact set to False during job termination (jobs_terminating.py:787), so 'only ever set to True' overstates it (though the probe-failure conclusion holds); `_get_job_plan` also requires `profile.instances is None` to add cloud offers; MIGRATIONS.md has no literal 'one table per migration' rule; and the proposed `delete_endpoints` signature omits the `user` param that the volumes precedent needs for event emission (and delete_volumes does not hint the pipeline). External-URL claims (recipes.vllm.ai, docs.sglang.io) were not network-verified but everything checkable locally, including PR #3856 being OPEN, was confirmed. - -## [minor] [PARTIALLY_WRONG] 2. Ground truth, item 5 -CLAIM: `registered` is never unset by later probe failures (verified: it's only ever set to `True` at `jobs_running.py:1158`) -CORRECTION: The parenthetical is false: `registered` IS also assigned `False` at src/dstack/_internal/server/background/pipeline_tasks/jobs_terminating.py:787 in `_unregister_replica_and_update_result` (result.job_update_map["registered"] = False) when a replica is unregistered during job termination. The load-bearing conclusion survives — probe failures never unset it (the True assignment at jobs_running.py:1158 in `_maybe_register_replica` is the only place it is set True, and the False path only runs in the terminating pipeline) — but the plan should not claim it is 'only ever set to True'. - -## [minor] [PARTIALLY_WRONG] 2. Ground truth, item 7 (and §7.3 step 2) -CLAIM: plan offers always include existing-instance offers, and add new-cloud offers only when profile.creation_policy == CreationPolicy.REUSE_OR_CREATE -CORRECTION: The actual condition in `_get_job_plan` (src/dstack/_internal/server/services/runs/plan.py:979) is `if profile.creation_policy == CreationPolicy.REUSE_OR_CREATE and profile.instances is None:` — backend offers are also suppressed when `profile.instances` is set, even under REUSE_OR_CREATE. The direction the plan relies on (creation_policy=reuse ⇒ offers are exclusively existing-instance offers) is correct, but 'non-empty offers ⇔ matches existing fleets' as stated omits the `instances` clause. - -## [minor] [PARTIALLY_WRONG] 4. DB model, migration, events — Migration bullet -CLAIM: Respect contributing/MIGRATIONS.md (one table per migration; this one is additive so zero-downtime-safe) -CORRECTION: contributing/MIGRATIONS.md contains no 'one table per migration' rule. The actual rule (section 'Altering multiple tables') is: 'Altering multiple tables should be done in separate transactions/migrations' — it applies to ALTERs (ACCESS EXCLUSIVE locks), not to creating a new table. The doc also recommends CREATE INDEX CONCURRENTLY for adding indexes (relevant only for existing tables). The plan's conclusion (a single additive create_table migration is zero-downtime-safe) is unaffected. The autogenerate command and versions// layout are exactly right (alembic.ini file_template = %(year)d/%(month).2d_... at src/dstack/_internal/server/alembic.ini:11). - -## [minor] [PARTIALLY_WRONG] 5.1 Service module — delete_endpoints -CLAIM: delete_endpoints(session, project, names): volumes-style lock-retry (10×) ...; set to_be_deleted=True; emit; commit; hint -CORRECTION: Two deltas vs the cited precedent `delete_volumes` (src/dstack/_internal/server/services/volumes.py:321): (1) its signature is `(session, project, names, user)` — `user` is required because the event is emitted with `events.UserActor.from_user(user)` (volumes.py:373); the plan's signature omits `user` yet mandates an emit. (2) `delete_volumes` does NOT hint the pipeline after commit (no pipeline_hinter parameter); adding a hint for endpoints is a reasonable improvement but is not 'volumes-style'. The lock-retry (range(10), lock_expires_at.is_(None), with_for_update(key_share=True), ServerClientError on failure, to_be_deleted=True) is confirmed verbatim. - -## [minor] [PARTIALLY_WRONG] 2. Ground truth, item 10 -CLAIM: The existing skills/dstack/SKILL.md (567 lines) targets a CLI-driving interactive agent -CORRECTION: skills/dstack/SKILL.md is 566 lines (wc -l; file ends with a trailing newline). PR #3856 being OPEN (not merged) and skills/dstack-dev not existing on master were both confirmed (gh pr view 3856 → state OPEN; no skills/dstack-dev directory). - -# Verifier 1 - -OVERALL: The pipeline-framework parts of the plan are accurate: the Pipeline/Fetcher/Worker/Heartbeater contracts, the volume fetch query it copies (including the last_processed_at == created_at fast path, lock_owner filter, and with_for_update(skip_locked=True, key_share=True, of=Model)), the InstanceFetcher per-status-interval precedent (instances/__init__.py:190-211 uses min_processing_interval * 2 for steady-state statuses, so the plan's ACTIVE-interval WHERE clause fits), the update-map helpers, and all section-9 settings/packaging/docs facts check out against the code. The one substantive error is the run-teardown shortcut: delete_runs raises ServerClientError for any non-finished run and stop_runs only flips the run to TERMINATING for asynchronous termination by RunPipeline, so the agent-path pre-cleanup in 6.2 step 3 (and the 8.3 stop_service tool) written as a one-shot 'stop_runs + delete_runs' will fail on active leftovers and, combined with the attempts-before-launch rule, can exhaust the attempt budget without ever running the agent — it must adopt the same two-iteration wait-until-finished pattern the plan already prescribes for _process_to_be_deleted_endpoint. Two nuances worth carrying into implementation: the interim token-guarded attempts UPDATE is mechanically consistent with the framework (it leaves lock_token/lock_expires_at untouched so the Heartbeater and final write are unaffected) but has no exact precedent — existing workers only commit mid-process for related-row lock claims (fleets.py, jobs_submitted.py), so the final update map must be careful not to write a stale attempts value over the interim one; and the 6.4 probe helper must replicate _execute_probe's except (SSHError, httpx.RequestError) → False clause or health-check failures will raise past the failure counter. - -## [major] [PARTIALLY_WRONG] 6.2 _process_provisioning_endpoint step 3 (agent pre-cleanup); also 8.3 stop_service tool -CLAIM: If a leftover run named after the endpoint exists (from a crashed attempt), terminate it first (stop_runs(abort=True) + delete_runs) so the agent starts clean. -CORRECTION: delete_runs cannot be called right after stop_runs when the leftover run is still active. src/dstack/_internal/server/services/runs/__init__.py::delete_runs (line 909) raises ServerClientError 'Cannot delete active runs: ...' (lines 932-936) for any run not in RunStatus.finished_statuses(); stop_runs (line 865) only sets termination_reason and switches the run to TERMINATING ('The run will be terminated by RunPipeline', line 903) — actual termination is asynchronous. TERMINATING is not a finished status (core/models/runs.py::RunStatus.finished_statuses = [TERMINATED, FAILED, DONE]). The one-shot sequence works only if the leftover run already finished; for an active leftover it raises, the exception is swallowed by Worker.start's generic handler (base.py:360-361), the final update-map write is skipped, and — since attempts is persisted before the agent launches — each such loop can burn an attempt without ever running the agent. Fix: mirror _process_to_be_deleted_endpoint's own (correct) design — stop_runs, then no-op until the run reaches a finished status on a later iteration, then delete_runs (or rely on submit_run's name-collision branch at runs/__init__.py:716, which itself calls delete_runs and has the same active-run constraint). The 8.3 stop_service agent tool ('runs.stop_runs(abort=True) + delete_runs') needs the same wait-until-finished handling. - -## [minor] [PARTIALLY_WRONG] 6.2 _process_to_be_deleted_endpoint -CLAIM: Stop and delete the backing run if present (stop_runs(abort=False), tolerate ResourceNotExistsError; then delete_runs once finished ...) -CORRECTION: The two-iteration stop-then-delete-once-finished flow is correct, but 'tolerate ResourceNotExistsError' targets the wrong exception: neither stop_runs nor delete_runs raises ResourceNotExistsError for missing run names — both just filter RunModel.run_name.in_(names) and silently no-op on zero matches (services/runs/__init__.py:873-880, 915-921). The error to guard against is ServerClientError from delete_runs when the run is not yet finished (runs/__init__.py:932-936). - -## [minor] [PARTIALLY_WRONG] 6.4 Direct model probe (health check) -CLAIM: Replicate _execute_probe (scheduled_tasks/probes.py:106-126) as check_model_endpoint(...) -> bool [snippet returns resp.is_success with no exception handling] -CORRECTION: The real _execute_probe wraps the whole tunnel+request in try/except (SSHError, httpx.RequestError) and returns False on failure (probes.py:112-126); it also passes follow_redirects=False. The plan's snippet omits the except clause, so an unreachable/dead replica raises SSHError (from get_service_replica_tunnel) or httpx.ConnectError instead of returning False; the Worker's generic exception handler (base.py:360-361) would then skip the endpoint's final update entirely — the health_check_failures counting path in 6.2 would never execute for exactly the failure mode it is meant to count. check_model_endpoint must catch (SSHError, httpx.RequestError) → False. Everything else in the snippet is accurate: get_service_replica_client(job: JobModel) is an async context manager (services/jobs/job_replica_http_client.py:22), the dummy 'http://dstack' host matches probes.py:116, the body/headers match _openai_model_probe_spec (services/jobs/configurators/base.py:472-493), orjson is a base dependency (pyproject.toml [project].dependencies), and the tunnel targets job_spec.service_port directly (job_replica_tunnel.py:25-43), so it is gateway-independent and needs no HTTP auth. - -## [minor] [PARTIALLY_WRONG] 2.3 Multi-replica safety (heartbeat semantics) -CLAIM: a Heartbeater extending lock_expires_at every ~1s while a worker processes an item -CORRECTION: The heartbeat loop runs every ~1s (heartbeat_delay=1.0, base.py:172,189), but it only writes lock_expires_at when the lease is within the heartbeat_trigger margin of expiry (base.py:213: item.lock_expires_at < now + self._hearbeat_margin). With lock_timeout=30s and heartbeat_trigger=15s each item's lease is extended roughly every ~15s, not every ~1s. The load-bearing part of the claim — indefinite extension for minutes-long process() calls, with untrack-on-missed-heartbeat and token-guarded writes as the backstop — is correct (base.py:204-254; precedent: instances/cloud_provisioning.py::create_cloud_instance offer loop). - -# Verifier 2 - -OVERALL: The plan's fetch/lock/cadence mechanics and its reading of submit_run's name-collision semantics are accurate, but the state machine has three blocker-level logic failures around retry and crash recovery: (1) the SUBMITTED handler has no reconcile step, so the advertised "run created but FK write lost" recovery actually ends in a terminal FAILED endpoint plus an orphaned, unstoppable service run; (2) the retry-after-run-failure branch never deletes the failed run, whose name keeps matching the reconcile fallback, making the "no run yet" retry branches unreachable for both preset and agent; (3) even fixed, PROVISIONING contains no preset re-submission path and no transition back to SUBMITTED, so preset retries no-op until the deadline. Two majors compound this: name-based reconcile/cleanup can adopt or destroy an unrelated pre-existing user run with the same name (the section 6.5 "submission fails cleanly" claim holds only for presets), and every "stop_runs + delete_runs" one-pass sequence in the plan always raises because stop is asynchronous and TERMINATING is not a finished status. All fixes are localized: reconcile-first in SUBMITTED with an ownership guard, delete the consumed run on retry, retry via status=SUBMITTED, and two-phase stop-then-delete everywhere. - -## [blocker] [WRONG] 6.1 / 6.3 (crash recovery of SUBMITTED preset submission) -CLAIM: "Run created but endpoint-row update lost the lock" is reconciled on the next iteration via run_id/run-name lookup (section 6.5). -CORRECTION: The reconcile-by-name lives only in _process_provisioning_endpoint (6.2 step 2). If the SUBMITTED handler's single token-guarded update {status: PROVISIONING, run_id, ...} is lost, the row is re-fetched still in SUBMITTED, and _process_submitted_endpoint (steps 1-4) has NO reconcile step — it re-runs preset matching and calls submit_run again with the same run name. submit_run (src/dstack/_internal/server/services/runs/__init__.py:716-718) unconditionally calls delete_runs on the colliding name, and delete_runs (:932-936) raises ServerClientError('Cannot delete active runs: ...') because the just-submitted run is SUBMITTED/PROVISIONING (not in RunStatus.finished_statuses(), core/models/runs.py:663). Per plan step 2 ('catch ServerClientError => FAILED'), the endpoint goes terminal FAILED while the orphaned service run keeps provisioning/running — and since run_id was never written and FAILED is terminal, neither the pipeline nor _process_to_be_deleted_endpoint ('backing run if present') will ever stop it. Minimal fix: make _process_submitted_endpoint reconcile-first — before matching/submitting, get_run_model_by_name(endpoint.name); if a non-deleted run exists that the endpoint plausibly owns (run.user_id == endpoint.user_id and run submitted_at >= endpoint.created_at), adopt it: write {status: PROVISIONING, run_id: run.id, provisioning_started_at: NOW, attempts: 1} and return. - -## [blocker] [WRONG] 6.2 step 2b (retry after backing-run failure) -CLAIM: Run in finished_statuses and attempts < MAX: clear run_id, stay PROVISIONING (next iteration re-runs preset/agent path). -CORRECTION: The failed run is never deleted, so it keeps existing non-deleted under the endpoint's name, and step 2's own fallback ('or a non-deleted run named after the endpoint exists') re-adopts it on EVERY subsequent iteration — get_run_model_by_name (runs/__init__.py:477-493) filters only deleted==False, so a FAILED run matches forever. The 'no run yet' branches (steps 3 and 4) are therefore unreachable after a failure: the handler loops finished-run -> clear run_id -> re-find-by-name until attempts hit MAX (if the state diagram's attempts++ applies per iteration) or the 1h deadline fires. Retry never actually re-provisions anything, for both preset and agent methods. Minimal fix: in the retry branch, soft-delete the consumed run (delete_runs in a fresh get_session_ctx — it is finished, so this succeeds) in the same iteration as clearing run_id, so the next iteration genuinely classifies as 'no run yet'. - -## [blocker] [WRONG] 6.2 step 4 (no run + method=preset) -CLAIM: Reaching here means the run vanished -> same retry logic as 2b (i.e., the preset path is re-run). -CORRECTION: There is no preset re-submission path reachable from PROVISIONING: preset submission exists only in _process_submitted_endpoint, step 3 of the PROVISIONING handler is explicitly gated on method=agent, and no transition ever returns the endpoint to SUBMITTED. So even with finding #2 fixed, a preset endpoint whose run failed sits in PROVISIONING as a pure no-op loop (10s cadence) until the 1h deadline, then FAILED 'Provisioning timed out' — the attempts=2 retry budget is dead code for presets. The plan also never defines whether a failed preset attempt may fall through to the agent (provisioning_method stays 'preset:', so as written it cannot). Minimal fix: on retry, transition back to {status: SUBMITTED, run_id: NULL, provisioning_method: NULL} (keeping attempts and provisioning_started_at) so the SUBMITTED dispatcher re-decides preset-vs-agent; state explicitly whether re-matching may pick the agent on attempt 2. - -## [major] [PARTIALLY_WRONG] 6.5 consequence (a) / 6.2 steps 2-3 (name-based reconcile vs pre-existing user run) -CLAIM: An existing active user run with the same name makes submission fail -> endpoint FAILED with a clear message. -CORRECTION: True only for the preset path (submit_run in SUBMITTED raises via delete_runs, runs/__init__.py:932-936). On the agent path no submission happens in SUBMITTED, so the endpoint enters PROVISIONING, where step 2's name-fallback ('or a non-deleted run named after the endpoint exists') adopts the user's unrelated pre-existing run as the backing run — if it is RUNNING+registered and probes OK, the endpoint goes ACTIVE bound to a run it does not own; worse, step 3's pre-launch cleanup ('if a leftover run named after the endpoint exists ... stop_runs(abort=True) + delete_runs') will abort and destroy the user's run. Nothing in create_endpoint (5.1) checks run-name collisions, so this is trivially reachable. Minimal fix: guard every name-based reconcile/cleanup with an ownership check (run.user_id == endpoint.user_id AND run submitted_at >= endpoint.created_at, or record the submitted run id in an interim token-guarded update before submit_run); alternatively reject/FAIL in SUBMITTED when a non-deleted foreign run already holds the name. - -## [major] [WRONG] 6.2 step 3 pre-launch cleanup / 8.3 stop_service tool -CLAIM: Terminate the leftover run first (stop_runs(abort=True) + delete_runs) so the agent starts clean. -CORRECTION: stop_runs is asynchronous: it only sets RunStatus.TERMINATING and defers to RunPipeline (runs/__init__.py:899-904, 'The run will be terminated by RunPipeline'). TERMINATING is not in finished_statuses() (core/models/runs.py:663-664), so an immediately-following delete_runs raises ServerClientError('Cannot delete active runs') every time — the one-pass 'stop + delete' sequence as written always fails, both in the pipeline pre-launch cleanup and in the agent's stop_service tool (8.3). Minimal fix: make it two-phase like _process_to_be_deleted_endpoint — issue stop_runs(abort=True), then treat 'leftover run still TERMINATING' as a no-op wait state (return, next iteration retries); only call delete_runs once the run reaches a finished status. Same fix applies inside the stop_service tool (poll with a short timeout before delete, or make delete best-effort). - -## [minor] [PARTIALLY_WRONG] 6.2 _process_to_be_deleted_endpoint (attack 5) -CLAIM: Deletion may take two iterations: first stop, then when the run reaches a finished status, delete it and mark the endpoint deleted. -CORRECTION: There is no bound on the wait: stop_runs(abort=False) is a graceful stop (TERMINATING set at runs/__init__.py:899-901, actual termination delegated to RunPipeline), and if the run is stuck TERMINATING the endpoint stays to_be_deleted forever, re-processed every 10s; the CLI's delete_configuration 'poll until gone' (5.3) never returns and the endpoint name is never freed (uniqueness is over non-deleted rows, section 4). Minimal fix: add a deletion deadline (e.g. reuse ENDPOINT_PROVISIONING_TIMEOUT or a fixed 10-15 min) after which the handler escalates to stop_runs(abort=True), and after a further grace period marks the endpoint deleted anyway with a status_message noting the run was left TERMINATING. - -## [minor] [PARTIALLY_WRONG] 4 / 6.5 (run_id FK ondelete=SET NULL, attack 7) -CLAIM: run_id: FK('runs.id', ondelete='SET NULL') — the authoritative link; reconcile handles the run going away. -CORRECTION: SET NULL is effectively dead: runs are only ever soft-deleted (delete_runs sets RunModel.deleted=True, runs/__init__.py:937-939; no server code hard-deletes RunModel rows — the only hard delete is the projects.id CASCADE, which also cascades the endpoint row itself). So after a user deletes the backing run, endpoint.run_id still resolves to a row with deleted=True, and the plan's PROVISIONING step 2 ('if run_id set ... load it') specifies no RunModel.deleted check — only the ACTIVE handler (step 1) mentions 'deleted'. It happens to behave acceptably (soft-deleted runs are always finished, so the retry branch fires), but the retry then interacts with finding #2 (the name-fallback correctly excludes deleted runs, so this path uniquely CAN reach 'no run yet'). Minimal fix: state explicitly that loading by run_id must treat run.deleted==True as 'run gone' in both PROVISIONING and ACTIVE handlers, and drop or annotate the SET NULL as project-cascade-only. - -# Verifier 3 - -OVERALL: Verified claude-agent-sdk 0.2.110 empirically in an isolated venv. The plan's §8 spike-hedged facts are mostly right (options fields, in-process MCP tools, bundled self-contained CLI binary, no node requirement, async IO), but three corrections matter: (1) BLOCKER — the `agent = ["claude-agent-sdk>=X"]` extra is uninstallable with dstack's `pydantic>=1.10.10,<2.0.0` pin (pyproject.toml:29) because mcp requires pydantic>=2.11; pip ResolutionImpossible verified, so the §11-Q4 plain-`anthropic` fallback (pydantic>=1.9,<3, compatible) becomes the mandatory v1 path unless dstack migrates to pydantic v2. (2) `ClaudeAgentOptions.env` merges over the full inherited server environment — there is no scrub mechanism; the §8.5 env-hygiene guarantee requires a custom Transport. (3) `allowed_tools` is a permission auto-approve list, not a toolset restriction — the hard no-Bash/no-filesystem boundary is `tools=[]` plus `allowed_tools` for the MCP tool names. - -## [blocker] [WRONG] 8.2 (packaging) / 9 / 11-Q4 -CLAIM: Packaging: new optional extra in pyproject.toml — agent = ["claude-agent-sdk>=X.Y"] (depends on dstack[server]), included in all -CORRECTION: The extra is not installable alongside dstack as pinned today. dstack pins `pydantic>=1.10.10,<2.0.0` (pyproject.toml:29), while claude-agent-sdk 0.2.110 requires `mcp>=1.23.0,<2.0.0` and mcp (1.28.1) requires `pydantic>=2.11.0,<3.0.0`. Verified empirically: `pip install --dry-run 'claude-agent-sdk>=0.2' 'pydantic>=1.10.10,<2.0.0'` fails with ResolutionImpossible. mcp has been a hard dependency since 0.1.0 and is required for the plan's in-process tools (`create_sdk_mcp_server` returns `McpSdkServerConfig`, `tool()` uses `mcp.types.ToolAnnotations`), so no SDK version avoids it. The §11 Q4 escape hatch — a plain `anthropic` tool-use loop — IS compatible (anthropic 0.116.0 requires `pydantic>=1.9.0,<3`), so it must be promoted from 'fallback' to the v1 default unless dstack migrates to pydantic v2 first. Also note mcp drags in pydantic-settings, starlette, uvicorn, jsonschema, pyjwt — a large transitive surface into the server env. - -## [major] [PARTIALLY_WRONG] 8.5 (env hygiene) -CLAIM: Env hygiene: the agent subprocess/client receives only ANTHROPIC_API_KEY (+ whatever the SDK minimally needs) — never the server's environment (verify the SDK's env-passing mechanism) -CORRECTION: `ClaudeAgentOptions.env` does NOT scrub — it merges OVER the full inherited server environment. Verified in claude_agent_sdk/_internal/transport/subprocess_cli.py::SubprocessCLITransport.connect (v0.2.110, lines 430-436): `inherited_env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}` then `process_env = {**inherited_env, "CLAUDE_CODE_ENTRYPOINT": "sdk-py", **self._options.env, ...}`. There is no option to start from a clean env; by default the CLI subprocess inherits DB credentials, cloud creds, encryption keys — the exact blast radius §8.3 tries to eliminate. Achieving the scrub requires a custom transport: `query()` and `ClaudeSDKClient` both accept `transport:` and `Transport` is a public ABC (connect/write/read_messages/close/is_ready/end_input), or subclass the internal `SubprocessCLITransport` and rebuild `process_env`. ANTHROPIC_API_KEY itself is consumed by the CLI subprocess from its env (pass via options.env); the Python SDK reads it only in _internal/session_resume.py. - -## [major] [PARTIALLY_WRONG] 8.2 / 8.3 (tool allowlist) -CLAIM: permissioning (allowed_tools) gives a hard tool allowlist; the agent gets only custom in-process dstack tools — no Bash, no filesystem tools; allowed_tools = exactly these mcp__dstack__* names -CORRECTION: `allowed_tools` maps to `--allowedTools` (subprocess_cli.py::_build_command), which is a permission auto-approve list — it does not remove built-in tools from the toolset. The hard toolset restriction is the separate `tools` field (`ClaudeAgentOptions.tools: list[str] | ToolsPreset | None`), which maps to `--tools`; `tools=[]` emits `--tools ""` giving an EMPTY base toolset (no Bash/Read/Write/WebFetch). The correct hard boundary is `tools=[]` (removes all built-ins) combined with `allowed_tools=["mcp__dstack__*" names]` (auto-approves the MCP tools so headless runs don't hit permission denials); optionally `disallowed_tools` as belt-and-suspenders. Also relevant: `setting_sources` defaults to None = no filesystem settings loaded, but passing `skills=` silently defaults setting_sources to ['user','project'] and appends 'Skill' to allowed_tools — do not use `skills` in the server context. - -# Verifier 4 - -OVERALL: Section 7's matching mechanism is factually sound: the _get_job_plan REUSE behavior, CreationPolicy literals, IDLE→is_available() semantics, str→OpenAIChatModel coercion, Env.update, the ProfileParams merge-loop precedent, Optional RunSpec.profile, and get_plan's commit-free/repo-free/plugin-safe behavior all check out at the cited symbols. The one correction that will break at runtime as written: the endpoint worker's refetch must eager-load ProjectModel.backends (volumes.py:237 pattern), because get_plan lazy-accesses project.backends and async SQLAlchemy raises MissingGreenlet otherwise. Secondary corrections: get_plan raises ServerClientError at match time (SSH key, invalid merged config) so find_matching needs per-candidate error handling, REUSE planning still enumerates cloud backend offers per candidate fleet (latency/side-effects in the background loop), and the ≥1-available-offer criterion under-checks capacity for multi-replica presets. - -## [major] [PARTIALLY_WRONG] §6.1 worker refetch + §7.3 matching via get_plan -CLAIM: The worker refetches the endpoint row 'with joinedloads (project, user, run)' and can then call find_matching→get_plan on its session -CORRECTION: get_plan reaches backends_services.get_project_backends(project), which iterates the lazy `project.backends` relationship (services/backends/__init__.py:281 in get_project_backends_with_models) — under async SQLAlchemy this raises MissingGreenlet unless eager-loaded. Every existing pipeline that reaches backend code chains the load, e.g. volumes pipeline: `.options(joinedload(VolumeModel.project).joinedload(ProjectModel.backends))` (background/pipeline_tasks/volumes.py:237); same pattern in gateways.py:232, jobs_submitted.py:821. The plan's joinedload list must include `EndpointModel.project → ProjectModel.backends` (or find_matching must load the project with backends in its own session). This bites whenever any candidate fleet is a cloud fleet: find_optimal_fleet_with_offers → _get_backend_offers_in_fleet → get_offers_by_requirements → get_project_backends (offers.py:44); only pure-SSH-fleet projects escape via check_can_create_new_cloud_instance_in_fleet raising ValueError (plan.py:716-723). - -## [minor] [PARTIALLY_WRONG] §7.3 — matching error handling / SSH key requirement -CLAIM: SSH-key requirement and ServerClientError handling only matter at submission (submit_run / register_service); matching just calls get_plan per candidate -CORRECTION: get_plan itself calls validate_run_spec_and_set_defaults (runs/__init__.py:544-548), which raises ServerClientError("ssh_key_pub must be set if the user has no ssh_public_key") at runs/spec.py:128-132 — so the SSH-key requirement bites at MATCH time, not just submission (mitigated: create_user always generates a keypair, services/users.py:176-186, but e.g. legacy/SSO-created users without keys would fail matching, and validate can also raise for a bad merged preset: probe caps runs/spec.py:107-117, volume mount paths :79-81, scheduled+scale-to-zero :100-106). find_matching must catch ServerClientError per candidate and treat it as no-match/skip, otherwise one bad preset aborts matching; the plan only specifies catching ServerClientError around submit_run (§6.2 step 2) and 'invalid files skipped' at YAML-parse level (§7.2). - -## [minor] [PARTIALLY_WRONG] §2.7 / §7.3 — cost of REUSE matching -CLAIM: get_plan with creation_policy=reuse is effectively an existing-fleet-only check -CORRECTION: The RETURNED offers are existing-instance-only (correct), but the planner still enumerates cloud backend offers for every candidate cloud fleet regardless of creation_policy: get_job_plans calls find_optimal_fleet_with_offers without skip_backend_offers_on_pool_capacity (plan.py:171-180), so `skip_backend_offers` is False (plan.py:405) and _get_backend_offers_in_fleet → get_offers_by_requirements runs per fleet (plan.py:416-424, offers.py:30+, hitting backend Compute.get_offers), then again uncapped for the optimal fleet (plan.py:449-456); the results are discarded by _get_job_plan under REUSE. Consequence for §7.3: each preset candidate × replica group evaluation in the background worker performs cloud-offer enumeration (network/catalog calls, seconds each) — harmless for correctness (heartbeater covers it) but not the light check the plan implies; worth noting or bounding candidate count. - -## [minor] [PARTIALLY_WRONG] §7.3 / §11.6 — match criterion for multi-replica presets -CLAIM: Match ⇔ every job_plan has ≥1 offer with offer.availability.is_available() -CORRECTION: For presets with replica groups where count.min > 1 this criterion under-checks capacity: get_job_plans plans replica_num=0 per replica group (plan.py:136-142), so one available instance satisfies the criterion even though get_nodes_required_num = sum of count.min (runs/spec.py:250-258) instances are needed. When no fleet has full pool capacity (has_pool_capacity requires nodes_required_num ≤ available instance offers, plan.py:393-399), the optimal fleet is chosen by price and can still contribute one available offer → 'match' for a service that cannot fully place on existing fleets. Acceptable given the plan calls matching advisory (fleet-drift note), but the criterion is 'each job can start', not 'the whole service fits'; either accept explicitly or check total available offers ≥ nodes_required_num. - -# Verifier 5 - -OVERALL: This is an unusually implementable plan: every code-level ground-truth claim I spot-checked against master@28ea5f86f confirmed exactly (submit_run/get_plan signatures, _get_job_plan's reuse-only offer behavior, pipeline_tasks/base.py symbols, PipelineModelMixin, Target.from_model, Env.update, RunModel.service_spec, the volumes lock-retry delete, the dual-core-model config pattern). The blocking problems are lifecycle-semantics holes concentrated in §6.2 — the preset retry path is circular and never re-submits, and FAILED transitions are inconsistent about backing-run teardown — plus two cross-section conflicts: the volumes-style delete API cannot acquire the lock during the minutes-long lease-held agent step, and the health probe's model_name/prefix inputs are unspecified with the obvious guess (endpoint config's model string) being wrong. Fixing the eight findings above, most of which are one-paragraph edits to §6.2/§8.3, would make M1-M5 implementable without re-research. - -## [blocker] [WRONG] §6.2 _process_provisioning_endpoint (steps 2 & 4) + state diagram -CLAIM: On backing-run failure with attempts < MAX: "clear run_id, stay PROVISIONING (next iteration re-runs preset/agent path)"; step 4 (no run + method=preset) says "same retry logic as 2b". -CORRECTION: This is circular: no step ever re-submits a preset run from PROVISIONING. Step 3 only handles method=agent; step 4 points back to 2b, which only clears run_id — so a preset endpoint whose run failed spins as a no-op until the 1h deadline. Also, the state diagram says "attempts++" on retry but the prose never says where attempts is incremented for the preset path (SUBMITTED sets attempts=1 only on first submission). Fix: add an explicit "no run yet + method=preset:*" step that increments attempts (token-guarded, like the agent path) and re-runs EndpointPresetService.find_matching + merged submit_run in a fresh session — and state whether re-matching may switch the method to agent when the preset no longer matches (recommend: yes, re-dispatch exactly as _process_submitted_endpoint does). Alternative: transition back to SUBMITTED and let _process_submitted_endpoint re-dispatch. Pick one and write it into §6.2. - -## [major] [WRONG] §5.1 delete_endpoints vs §6.2 step 3 / §8.5 -CLAIM: delete_endpoints uses the volumes-style lock-retry (10×, ServerClientError if rows are pipeline-locked), while the agent step legitimately holds the heartbeated lease for minutes up to the 1h deadline. -CORRECTION: These two settled decisions contradict each other: the volumes pattern (services/volumes.py::delete_volumes — retries 10×0.5s for lock_expires_at IS NULL, then raises "volumes are being processed currently") assumes short processing steps, but §6.2's in-worker agent run keeps lock_expires_at perpetually extended, so `dstack delete` on a PROVISIONING(agent) endpoint deterministically fails for the entire agent run. Fix: for endpoints, set to_be_deleted=True with a plain UPDATE that does not require acquiring the pipeline lock (safe — the flag is only consumed by the pipeline; the worker's final token-guarded write should re-check it), and have the agent step periodically check to_be_deleted and abort. If instead you keep the volumes pattern, the plan must say deletion is rejected during agent provisioning and the CLI must surface that. - -## [major] [PARTIALLY_WRONG] §6.4 Direct model probe -CLAIM: check_model_endpoint(job_model, model_name, prefix="/v1") — the plan never says where model_name and prefix come from. -CORRECTION: The mechanism (get_service_replica_client + dummy host) is verified correct, but the inputs are undefined and the obvious guess — endpoint configuration's `model` string — is wrong: §7.3 matches presets case-insensitively and the agent may serve under a different name (e.g. --served-model-name), while chat/completions model routing is name-sensitive; prefix is also not always /v1. Specify: model_name = the backing run's ServiceSpec.model.name (RunModel.service_spec, server/models.py:446; ServiceModelSpec, core/models/runs.py:635 — note it has name/base_url/type, no prefix field) and prefix = the backing service configuration's model.prefix (OpenAIChatModel.prefix, core/models/services.py:72). - -## [major] [PARTIALLY_WRONG] §8.3 submit_service tool / §1 deliverable -CLAIM: The endpoint's deliverable URL comes from ServiceSpec.model.base_url and activation requires a model probe, but nothing enforces that the agent-submitted service YAML declares `model:` — only prompt guidance in §8.4. -CORRECTION: An agent deployment without `model:` can reach RUNNING + registered, leaving ServiceSpec.model = None — Endpoint.url stays None and the health probe has no model name, so activation semantics are undefined. Add a hard check to the submit_service tool: reject (or auto-inject with the endpoint's model value) service YAML whose `model:` is unset. Note it in the §8.3 table and add it to M4's tests. (Presets are already safe: §7.3 matching requires configuration.model.name.) - -## [major] [PARTIALLY_WRONG] §6.2 ACTIVE→FAILED and agent-exhausted FAILED transitions -CLAIM: Only the deadline-exceeded branch says "stop the backing run if any"; the health-check FAILED ("Model endpoint stopped responding") and the final agent-failure FAILED say nothing about the backing run, and FAILED rows are terminal (excluded from the fetcher). -CORRECTION: In both unstated cases a GPU-consuming service run can still be RUNNING and will leak indefinitely, since FAILED endpoints are never re-fetched and agent cleanup only happens before the *next* attempt. State explicitly, per FAILED transition, whether the backing run is stopped. Recommended: stop_runs on every FAILED transition, matching the deadline branch. If deliberately left running (e.g. for debugging), document that choice and that deleting the FAILED endpoint still stops+deletes the run via _process_to_be_deleted_endpoint. - -## [minor] [PARTIALLY_WRONG] §3 EndpointSpec vs §5.2 CreateEndpointRequest -CLAIM: EndpointSpec{configuration, configuration_path} is defined in core/models/endpoints.py, but CreateEndpointRequest takes bare `configuration: EndpointConfiguration` and nothing in v1 consumes EndpointSpec (only §12's Later plugins item references it). -CORRECTION: A dead type invites the implementer to guess whether configuration_path should be wired through create/CLI. Either drop EndpointSpec from the v1 file (introduce it with the Later plugins work), or change CreateEndpointRequest to take `spec: EndpointSpec` and have the CLI configurator populate configuration_path (RunSpec precedent). State which. - -## [minor] [PARTIALLY_WRONG] §6.2 _process_provisioning_endpoint step 2 -CLAIM: "probe OK ⇒ ACTIVE (reset health_check_failures), record URL source" -CORRECTION: "record URL source" is an undefined operation: §4's EndpointModel has no url column and §6.1's _EndpointUpdateMap has no url field, while §5.1 says url is derived at read time from the joined run's service_spec. Either delete the phrase (nothing needs recording if url is always read-time-derived) or, if something must be persisted, add the column to §4 and the field to the update map. - -## [minor] [PARTIALLY_WRONG] §10 M2 vs §9 settings -CLAIM: M2 implements "deadline/attempts" and ACTIVE monitoring, but no milestone includes adding DSTACK_SERVER_ENDPOINT_PROVISIONING_TIMEOUT or the code-level constants (ENDPOINT_PROVISIONING_MAX_ATTEMPTS, ENDPOINT_ACTIVE_CHECK_INTERVAL, ENDPOINT_HEALTH_CHECK_MAX_FAILURES) to server/settings.py — M3 covers only ENDPOINT_PRESETS_DIR and M4 only the agent settings. -CORRECTION: Add to the M2 bullet: "settings.py: ENDPOINT_PROVISIONING_TIMEOUT env var (+ env.md entry per the module docstring rule) and the three code-level constants". While there, reconcile §9 calling ENDPOINT_PROVISIONING_MAX_ATTEMPTS a "code-level constant (not env)" with §6.2 referencing it as settings.ENDPOINT_PROVISIONING_MAX_ATTEMPTS — say they live as plain module constants in settings.py. diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py index 51b016ecbe..17dd39b535 100644 --- a/src/dstack/_internal/cli/commands/endpoint.py +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -1,5 +1,4 @@ import argparse -import sys import time from typing import Iterable @@ -10,7 +9,11 @@ EndpointNameCompleter, EndpointPresetNameCompleter, ) -from dstack._internal.cli.services.endpoint_logs import EndpointLogPoller +from dstack._internal.cli.services.endpoint_logs import ( + EndpointLogPoller, + EndpointLogRecord, + print_endpoint_log, +) from dstack._internal.cli.utils.common import ( LIVE_TABLE_PROVISION_INTERVAL_SECS, LIVE_TABLE_REFRESH_RATE_PER_SEC, @@ -224,14 +227,13 @@ def _logs(self, args: argparse.Namespace): for log in self._get_endpoint_logs( endpoint=endpoint, start_time=start_time, watch=args.watch ): - sys.stdout.buffer.write(log) - sys.stdout.buffer.flush() + print_endpoint_log(log) except KeyboardInterrupt: pass def _get_endpoint_logs( self, endpoint: Endpoint, start_time, watch: bool = False - ) -> Iterable[bytes]: + ) -> Iterable[EndpointLogRecord]: poller = EndpointLogPoller(api=self.api, endpoint=endpoint, start_time=start_time) while True: yield from poller.poll() diff --git a/src/dstack/_internal/cli/services/configurators/endpoint.py b/src/dstack/_internal/cli/services/configurators/endpoint.py index 5368821982..3afd8ab81a 100644 --- a/src/dstack/_internal/cli/services/configurators/endpoint.py +++ b/src/dstack/_internal/cli/services/configurators/endpoint.py @@ -9,7 +9,11 @@ ApplyEnvVarsConfiguratorMixin, BaseApplyConfigurator, ) -from dstack._internal.cli.services.endpoint_logs import EndpointLogPoller +from dstack._internal.cli.services.endpoint_logs import ( + EndpointLogPoller, + EndpointLogRecord, + print_endpoint_log, +) from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args from dstack._internal.cli.utils.common import ( NO_OFFERS_WARNING, @@ -215,9 +219,9 @@ def _follow_endpoint_apply(self, endpoint: Endpoint): exit(1) -def _print_endpoint_log_batch(logs: list[bytes]) -> None: +def _print_endpoint_log_batch(logs: list[EndpointLogRecord]) -> None: for log in logs: - console.out(log.decode(errors="replace"), end="") + print_endpoint_log(log) def _apply_profile(conf: EndpointConfiguration, profile: Profile): diff --git a/src/dstack/_internal/cli/services/endpoint_logs.py b/src/dstack/_internal/cli/services/endpoint_logs.py index d629c84133..fe0623b10e 100644 --- a/src/dstack/_internal/cli/services/endpoint_logs.py +++ b/src/dstack/_internal/cli/services/endpoint_logs.py @@ -1,8 +1,12 @@ import base64 from collections import Counter +from dataclasses import dataclass from datetime import datetime, timedelta from typing import Optional +from rich.text import Text + +from dstack._internal.cli.utils.common import console from dstack._internal.core.models.endpoints import Endpoint from dstack._internal.core.models.logs import LogEvent from dstack._internal.server.schemas.logs import PollLogsRequest @@ -10,6 +14,12 @@ _ENDPOINT_LOG_WATCH_OVERLAP = timedelta(seconds=20) +@dataclass(frozen=True) +class EndpointLogRecord: + timestamp: datetime + message: str + + class EndpointLogPoller: def __init__(self, *, api, endpoint: Endpoint, start_time: Optional[datetime] = None) -> None: self._api = api @@ -18,8 +28,8 @@ def __init__(self, *, api, endpoint: Endpoint, start_time: Optional[datetime] = self._latest_printed_at: Optional[datetime] = None self._seen_log_counts: Counter[tuple[datetime, str, str]] = Counter() - def poll(self) -> list[bytes]: - logs: list[bytes] = [] + def poll(self) -> list[EndpointLogRecord]: + logs: list[EndpointLogRecord] = [] next_token = None batch_log_counts: Counter[tuple[datetime, str, str]] = Counter() while True: @@ -39,7 +49,7 @@ def poll(self) -> list[bytes]: for log in resp.logs: if not self._should_print(log, batch_log_counts=batch_log_counts): continue - logs.append(_format_log(log)) + logs.append(_decode_log(log)) next_token = resp.next_token if next_token is None: break @@ -71,6 +81,17 @@ def _log_key(log: LogEvent) -> tuple[datetime, str, str]: return (log.timestamp, log.log_source.value, log.message) -def _format_log(log: LogEvent) -> bytes: +def print_endpoint_log(log: EndpointLogRecord) -> None: timestamp = log.timestamp.astimezone().strftime("%Y-%m-%d %H:%M:%S") - return f"[{timestamp}] ".encode() + base64.b64decode(log.message) + console.print( + Text(f"[{timestamp}]", style="log.time"), + Text(log.message.rstrip("\r\n"), style="log.message"), + soft_wrap=True, + ) + + +def _decode_log(log: LogEvent) -> EndpointLogRecord: + return EndpointLogRecord( + timestamp=log.timestamp, + message=base64.b64decode(log.message).decode(errors="replace"), + ) diff --git a/src/tests/_internal/cli/commands/test_endpoint.py b/src/tests/_internal/cli/commands/test_endpoint.py index 07de32e142..74cea4dbad 100644 --- a/src/tests/_internal/cli/commands/test_endpoint.py +++ b/src/tests/_internal/cli/commands/test_endpoint.py @@ -90,8 +90,7 @@ def test_reads_endpoint_logs(self): logs = list(command._get_endpoint_logs(endpoint=endpoint, start_time=None)) assert len(logs) == 1 - assert logs[0].startswith(b"[") - assert logs[0].endswith(b"] agent log\n") + assert logs[0].message == "agent log\n" request = command.api.client.logs.requests[0] assert request.run_name == endpoint.name assert request.job_submission_id == endpoint.id @@ -131,8 +130,8 @@ def test_watch_endpoint_logs_does_not_swallow_same_timestamp_logs(self, monkeypa logs = command._get_endpoint_logs(endpoint=endpoint, start_time=None, watch=True) - assert next(logs).endswith(b"] first\n") - assert next(logs).endswith(b"] second\n") + assert next(logs).message == "first\n" + assert next(logs).message == "second\n" def test_stop_endpoint(self): endpoint = _get_endpoint() From 8306f5e707b4fae34548151d78f3c6d4b5f49226 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 8 Jul 2026 19:02:15 +0200 Subject: [PATCH 18/21] Support existing Claude auth for endpoint agent --- mkdocs/docs/reference/env.md | 8 + .../services/endpoints/agent/__init__.py | 8 +- .../server/services/endpoints/agent/claude.py | 75 +++++++- .../agent/resources/system_prompt.md | 4 + src/dstack/_internal/server/settings.py | 8 + .../services/endpoints/test_claude_agent.py | 161 +++++++++++++++++- 6 files changed, 256 insertions(+), 8 deletions(-) diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 86a7dd051d..762f4da461 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -162,6 +162,14 @@ slows down processing and may cause CPU spikes due to frequent SSH-connection es `https://dstack.example.com/{arch}/{version}/dstack-runner`. * `DSTACK_SHIM_DOWNLOAD_URL` – Overrides `dstack-shim` binary download URL. The URL can contain `{version}` and/or `{arch}` placeholders, see `DSTACK_RUNNER_DOWNLOAD_URL` for the details. + * `DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH` – Allows the endpoint agent to use + the server user's existing Claude CLI authentication when + `DSTACK_AGENT_ANTHROPIC_API_KEY` is not set. This is only for local + development. Do not set it together with `DSTACK_AGENT_ANTHROPIC_API_KEY`. + Production servers must use `DSTACK_AGENT_ANTHROPIC_API_KEY`. + This mode runs Claude without `--bare`, so Claude may read the server + user's Claude CLI auth and settings. It also passes the server process + `USER` to Claude because existing Claude CLI auth requires it. * `DSTACK_DEFAULT_CREDS_DISABLED` – Disables default credentials detection if set. Defaults to `None`. ## CLI diff --git a/src/dstack/_internal/server/services/endpoints/agent/__init__.py b/src/dstack/_internal/server/services/endpoints/agent/__init__.py index ef05a324bc..0b20911db7 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/agent/__init__.py @@ -73,7 +73,7 @@ async def abort_agent_endpoint(endpoint_model: EndpointModel) -> bool: def get_agent_service() -> AgentService: - if settings.AGENT_ANTHROPIC_API_KEY: + if _has_claude_agent_auth(): from dstack._internal.server.services.endpoints.agent.claude import ( ClaudeAgentService, get_claude_agent_unavailable_reason, @@ -87,10 +87,14 @@ def get_agent_service() -> AgentService: def get_agent_unavailable_reason() -> Optional[str]: - if not settings.AGENT_ANTHROPIC_API_KEY: + if not _has_claude_agent_auth(): return "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." from dstack._internal.server.services.endpoints.agent.claude import ( get_claude_agent_unavailable_reason, ) return get_claude_agent_unavailable_reason() + + +def _has_claude_agent_auth() -> bool: + return bool(settings.AGENT_ANTHROPIC_API_KEY or settings.AGENT_CLAUDE_USE_EXISTING_AUTH) diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index d114d53921..63de0327f5 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -723,8 +723,16 @@ def _load_agent_error(workspace: "_AgentWorkspace") -> Optional[str]: def get_claude_agent_unavailable_reason() -> Optional[str]: - if not settings.AGENT_ANTHROPIC_API_KEY: - return "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + if settings.AGENT_ANTHROPIC_API_KEY and settings.AGENT_CLAUDE_USE_EXISTING_AUTH: + return ( + "DSTACK_AGENT_ANTHROPIC_API_KEY and " + "DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH cannot both be set." + ) + if not settings.AGENT_ANTHROPIC_API_KEY and not _should_use_existing_claude_auth(): + return ( + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set. Set it or opt in to existing " + "Claude CLI auth with DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH=1." + ) if _get_claude_executable() is None: if settings.AGENT_CLAUDE_PATH is not None: return ( @@ -738,6 +746,9 @@ def get_claude_agent_unavailable_reason() -> Optional[str]: return None +_existing_auth_warning_logged = False + + class _AgentWorkspace: def __init__( self, @@ -750,6 +761,7 @@ def __init__( redacted_values: Sequence[str], endpoint_name: str, model: str, + dstack_home_dir: Optional[Path] = None, endpoint_constraints: str = "", max_price: Optional[float] = None, spot_policy: Optional[str] = None, @@ -758,6 +770,7 @@ def __init__( ) -> None: self.root_dir = root_dir self.home_dir = home_dir + self.dstack_home_dir = dstack_home_dir or home_dir self.work_dir = work_dir self.trace_path = trace_path self.env = env @@ -854,13 +867,14 @@ def _prepare_workspace( ) env = _build_agent_env( - home_dir=agent_home_dir, + home_dir=_get_claude_process_home_dir(agent_home_dir), project_name=endpoint_model.project.name, server_url=settings.SERVER_URL, token=token, endpoint_env=endpoint_env, bin_dir=bin_dir, ) + _warn_if_using_existing_claude_auth() redacted_values = _get_redacted_values( [ token, @@ -873,6 +887,7 @@ def _prepare_workspace( workspace = _AgentWorkspace( root_dir=root_dir, home_dir=home_dir, + dstack_home_dir=agent_home_dir, work_dir=work_dir, trace_path=trace_path, env=env, @@ -915,6 +930,12 @@ def _prepare_agent_home_dir(*, root_dir: Path, home_dir: Path) -> Path: return short_home_dir +def _get_claude_process_home_dir(agent_home_dir: Path) -> Path: + if _should_use_existing_claude_auth(): + return Path.home() + return agent_home_dir + + def _get_short_agent_home_dir(root_dir: Path) -> Path: root_hash = hashlib.sha1(str(root_dir).encode()).hexdigest()[:8] return Path("/tmp") / f"dah-{root_hash}" @@ -1112,13 +1133,16 @@ def _build_agent_env( env["PATH"] = str(bin_dir) if not path else os.pathsep.join([str(bin_dir), path]) env.update( { - "ANTHROPIC_API_KEY": settings.AGENT_ANTHROPIC_API_KEY or "", "HOME": str(home_dir), "DSTACK_PROJECT": project_name, "DSTACK_ENDPOINT_SERVER_URL": server_url, "DSTACK_ENDPOINT_BEARER_TOKEN": token, } ) + if settings.AGENT_ANTHROPIC_API_KEY: + env["ANTHROPIC_API_KEY"] = settings.AGENT_ANTHROPIC_API_KEY + elif _should_use_existing_claude_auth() and os.environ.get("USER"): + env["USER"] = os.environ["USER"] env.update(endpoint_env) return env @@ -1214,6 +1238,9 @@ def _install_agent_helper_scripts(workspace: _AgentWorkspace) -> None: scripts = { "progress": _get_agent_progress_script(), } + if _should_use_existing_claude_auth(): + scripts["dstack"] = _get_agent_home_script("dstack", workspace.dstack_home_dir) + scripts["ssh"] = _get_agent_home_script("ssh", workspace.dstack_home_dir) for name, script in scripts.items(): script_path = bin_dir / name script_path.write_text(script, encoding="utf-8") @@ -1249,6 +1276,24 @@ def main(): """ +def _get_agent_home_script(command: str, home_dir: Path) -> str: + real_command = shutil.which(command) + if real_command is None: + return f"""#!{sys.executable} +import sys + +print("Endpoint agent could not find the real {command} executable.", file=sys.stderr) +raise SystemExit(127) +""" + return f"""#!{sys.executable} +import os +import sys + +os.environ["HOME"] = {json.dumps(str(home_dir))} +os.execv({json.dumps(real_command)}, [{json.dumps(real_command)}, *sys.argv[1:]]) +""" + + def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: return { "prompt": _build_prompt(workspace), @@ -1450,7 +1495,6 @@ def _build_claude_command(request: dict[str, Any]) -> list[str]: cmd = [ _get_claude_executable() or "claude", "-p", - "--bare", "--output-format", "stream-json", "--verbose", @@ -1468,6 +1512,10 @@ def _build_claude_command(request: dict[str, Any]) -> list[str]: json.dumps(options["json_schema"]), request["prompt"], ] + if not _should_use_existing_claude_auth(): + cmd[2:2] = ["--bare"] + else: + cmd[2:2] = ["--setting-sources", "project,local"] if options["max_turns"] is not None: cmd[2:2] = ["--max-turns", str(options["max_turns"])] return cmd @@ -1479,6 +1527,23 @@ def _get_claude_executable() -> Optional[str]: return shutil.which("claude") +def _should_use_existing_claude_auth() -> bool: + return settings.AGENT_CLAUDE_USE_EXISTING_AUTH and not settings.AGENT_ANTHROPIC_API_KEY + + +def _warn_if_using_existing_claude_auth() -> None: + global _existing_auth_warning_logged + if not _should_use_existing_claude_auth() or _existing_auth_warning_logged: + return + logger.warning( + "DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH=1 is enabled. This mode is intended " + "only for local development. Production servers must set " + "DSTACK_AGENT_ANTHROPIC_API_KEY. Claude will run without --bare and may read " + "the server user's Claude CLI auth and settings." + ) + _existing_auth_warning_logged = True + + async def _read_agent_stdout( stream: asyncio.StreamReader, workspace: _AgentWorkspace, diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md index 0672485fd9..69992108e4 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md @@ -60,6 +60,10 @@ Do not rewrite previous `submissions.jsonl` lines; the latest line for a run is the current record. Stop runs you no longer need unless they are still needed for attach/SSH debugging, logs, or backend diagnosis. +After stopping a task or service, do not wait for it to disappear from +`dstack ps`. Confirm that the run reached a terminal status such as `stopped`, +`terminated`, or `failed`, then continue. + # Run Names Do not use the endpoint name itself as a submitted run name. diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 4e6f95c4e5..37d95dd7d7 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -33,6 +33,14 @@ def _getenv_non_empty(name: str) -> str | None: AGENT_ANTHROPIC_API_KEY = _getenv_non_empty("DSTACK_AGENT_ANTHROPIC_API_KEY") AGENT_CLAUDE_PATH = _getenv_non_empty("DSTACK_AGENT_CLAUDE_PATH") +# Development-only opt-in. Do not set together with +# DSTACK_AGENT_ANTHROPIC_API_KEY. Production servers should set +# DSTACK_AGENT_ANTHROPIC_API_KEY instead. Existing Claude CLI auth requires +# running without `claude --bare`, so the agent may read the server user's +# Claude CLI auth/settings. +AGENT_CLAUDE_USE_EXISTING_AUTH = environ.get_bool( + "DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH", default=False +) AGENT_ANTHROPIC_MODEL = os.getenv("DSTACK_AGENT_ANTHROPIC_MODEL", "claude-opus-4-8") DATABASE_URL = os.getenv( diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index 8b82eb22de..831a848317 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -490,9 +490,12 @@ async def runner(workspace, request): agent_home = Path(env["HOME"]) assert agent_home != session_root / "home" assert agent_home.resolve() == session_root / "home" + assert captured["workspace"].home_dir == session_root / "home" + assert captured["workspace"].dstack_home_dir == agent_home control_sock_path = agent_home / ".dstack" / "ssh" / f"{'x' * 60}.control.sock" assert len(str(control_sock_path)) < 104 assert stat.S_IMODE((session_root / "home" / ".ssh").stat().st_mode) == 0o700 + assert not (session_root / "bin" / "dstack").exists() assert env["DSTACK_PROJECT"] == "main" assert env["DSTACK_ENDPOINT_SERVER_URL"] == "http://127.0.0.1:8000" assert env["DSTACK_ENDPOINT_BEARER_TOKEN"] == "user-token" @@ -508,6 +511,8 @@ async def runner(workspace, request): assert '"format"' not in schema_text command = _build_claude_command(captured["request"]) assert command[0] == claude_path + assert "--bare" in command + assert "--setting-sources" not in command assert command[command.index("--tools") + 1] == captured["request"]["options"]["tools"] assert "--permission-mode" in command assert command[command.index("--permission-mode") + 1] == "bypassPermissions" @@ -536,6 +541,7 @@ async def runner(workspace, request): progress_helper = session_root / "bin" / "progress" assert progress_helper.exists() assert not (session_root / "bin" / "dstack").exists() + assert not (session_root / "bin" / "ssh").exists() progress_result = subprocess.run( [str(progress_helper), "Checked model config."], cwd=work_dir, @@ -555,6 +561,61 @@ async def runner(workspace, request): assert final_report["success"] is True assert final_report["run_name"] == "qwen-endpoint-1" + @pytest.mark.asyncio + async def test_existing_claude_auth_uses_real_home_for_claude_and_short_home_for_dstack( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", None) + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", True) + monkeypatch.setattr(claude_module, "_existing_auth_warning_logged", False) + _configure_fake_claude(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "SERVER_URL", "http://127.0.0.1:8000") + captured = {} + run_id = uuid.uuid4() + + async def runner(workspace, request): + captured["workspace"] = workspace + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=True, + run_id=run_id, + run_name="qwen-endpoint-1", + service_yaml="type: service\nname: qwen-endpoint-1\n", + ) + ) + + endpoint_model = await _create_endpoint_model(session) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.error is None + session_root = tmp_path / str(endpoint_model.id) / "1" + env = captured["request"]["env"] + workspace = captured["workspace"] + assert env["HOME"] == str(Path.home()) + assert "ANTHROPIC_API_KEY" not in env + assert workspace.home_dir == session_root / "home" + assert workspace.dstack_home_dir != workspace.home_dir + assert workspace.dstack_home_dir.resolve() == workspace.home_dir + wrapper = session_root / "bin" / "dstack" + assert wrapper.exists() + wrapper_text = wrapper.read_text() + assert f'os.environ["HOME"] = "{workspace.dstack_home_dir}"' in wrapper_text + assert str(workspace.home_dir) not in wrapper_text + ssh_wrapper = session_root / "bin" / "ssh" + assert ssh_wrapper.exists() + ssh_wrapper_text = ssh_wrapper.read_text() + assert f'os.environ["HOME"] = "{workspace.dstack_home_dir}"' in ssh_wrapper_text + assert str(workspace.home_dir) not in ssh_wrapper_text + command = _build_claude_command(captured["request"]) + assert "--bare" not in command + assert command[command.index("--setting-sources") + 1] == "project,local" + @pytest.mark.asyncio async def test_includes_endpoint_profile_constraints_in_prompt( self, @@ -784,11 +845,109 @@ def test_returns_error_when_configured_claude_path_is_not_executable( def test_returns_error_when_agent_key_is_blank(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "") + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", False) assert get_claude_agent_unavailable_reason() == ( - "DSTACK_AGENT_ANTHROPIC_API_KEY is not set." + "DSTACK_AGENT_ANTHROPIC_API_KEY is not set. Set it or opt in to existing " + "Claude CLI auth with DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH=1." ) + def test_existing_claude_auth_opt_in_enables_agent_without_api_key( + self, tmp_path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", None) + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", None) + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", True) + monkeypatch.setenv("USER", "server-user") + monkeypatch.setattr( + claude_module.shutil, + "which", + lambda name: f"/usr/local/bin/{name}" if name in {"claude", "dstack", "ssh"} else None, + ) + + command = _build_claude_command( + { + "prompt": "prompt", + "options": { + "allowed_tools": "Bash", + "disallowed_tools": "", + "model": "test-model", + "max_turns": None, + "json_schema": {}, + }, + } + ) + env = claude_module._build_agent_env( + home_dir=claude_module._get_claude_process_home_dir(Path("/tmp/agent-home")), + project_name="main", + server_url="http://127.0.0.1:8000", + token="token", + endpoint_env={}, + bin_dir=Path("/tmp/agent-bin"), + ) + dstack_home_dir = tmp_path / "short-home" + workspace = _AgentWorkspace( + root_dir=tmp_path, + home_dir=tmp_path / "home", + work_dir=tmp_path / "work", + trace_path=None, + env=env, + redacted_values=[], + endpoint_name="qwen-endpoint", + model="Qwen/Qwen3-0.6B", + dstack_home_dir=dstack_home_dir, + ) + + assert get_claude_agent_unavailable_reason() is None + assert command[0] == "/usr/local/bin/claude" + assert "--bare" not in command + assert command[command.index("--setting-sources") + 1] == "project,local" + assert env["HOME"] == str(Path.home()) + assert env["USER"] == "server-user" + assert "ANTHROPIC_API_KEY" not in env + claude_module._install_agent_helper_scripts(workspace) + wrapper = tmp_path / "bin" / "dstack" + assert wrapper.exists() + wrapper_text = wrapper.read_text() + assert str(dstack_home_dir) in wrapper_text + assert str(tmp_path / "home") not in wrapper_text + ssh_wrapper = tmp_path / "bin" / "ssh" + assert ssh_wrapper.exists() + ssh_wrapper_text = ssh_wrapper.read_text() + assert str(dstack_home_dir) in ssh_wrapper_text + assert str(tmp_path / "home") not in ssh_wrapper_text + + def test_returns_error_when_api_key_and_existing_claude_auth_are_both_set( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", None) + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", True) + + assert get_claude_agent_unavailable_reason() == ( + "DSTACK_AGENT_ANTHROPIC_API_KEY and " + "DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH cannot both be set." + ) + + def test_existing_claude_auth_logs_development_warning_once( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", None) + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", True) + monkeypatch.setattr(claude_module, "_existing_auth_warning_logged", False) + + claude_module._warn_if_using_existing_claude_auth() + claude_module._warn_if_using_existing_claude_auth() + + messages = [ + record.message + for record in caplog.records + if "DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH=1 is enabled" in record.message + ] + assert len(messages) == 1 + assert "only for local development" in messages[0] + assert "Production servers must set DSTACK_AGENT_ANTHROPIC_API_KEY" in messages[0] + def test_falls_back_to_claude_in_path(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(settings, "AGENT_CLAUDE_PATH", None) monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") From cd61651bf3898c034808ae1a4a89f6bd0e1f670e Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 8 Jul 2026 19:47:49 +0200 Subject: [PATCH 19/21] Support Claude effort for endpoint agent --- mkdocs/docs/reference/env.md | 4 ++ .../server/services/endpoints/agent/claude.py | 25 +++++++++-- src/dstack/_internal/server/settings.py | 1 + .../services/endpoints/test_claude_agent.py | 44 +++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 762f4da461..4a3ab58f85 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -170,6 +170,10 @@ slows down processing and may cause CPU spikes due to frequent SSH-connection es This mode runs Claude without `--bare`, so Claude may read the server user's Claude CLI auth and settings. It also passes the server process `USER` to Claude because existing Claude CLI auth requires it. + * `DSTACK_AGENT_CLAUDE_EFFORT` – Passes Claude CLI `--effort` for endpoint + agent sessions. Supported values are `low`, `medium`, `high`, `xhigh`, + and `max`. If unset, dstack does not pass `--effort` and Claude CLI uses + its default. * `DSTACK_DEFAULT_CREDS_DISABLED` – Disables default credentials detection if set. Defaults to `None`. ## CLI diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index 63de0327f5..134cd44667 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -80,6 +80,7 @@ _AGENT_PROGRESS_POLL_SECONDS = 1.0 _AGENT_PROCESS_ABORT_GRACE_SECONDS = 1.0 _CLAUDE_AGENT_TOOLS = "Bash,Read,Write,Edit,WebFetch,WebSearch,StructuredOutput" +_CLAUDE_EFFORT_LEVELS = ("low", "medium", "high", "xhigh", "max") _AGENT_ABORT_MESSAGE = "Endpoint stop requested" _AGENT_BIN_DIR_NAME = "bin" _AGENT_PROGRESS_ENV = "DSTACK_ENDPOINT_PROGRESS_LOG" @@ -733,6 +734,11 @@ def get_claude_agent_unavailable_reason() -> Optional[str]: "DSTACK_AGENT_ANTHROPIC_API_KEY is not set. Set it or opt in to existing " "Claude CLI auth with DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH=1." ) + if ( + settings.AGENT_CLAUDE_EFFORT is not None + and settings.AGENT_CLAUDE_EFFORT not in _CLAUDE_EFFORT_LEVELS + ): + return f"DSTACK_AGENT_CLAUDE_EFFORT must be one of: {', '.join(_CLAUDE_EFFORT_LEVELS)}." if _get_claude_executable() is None: if settings.AGENT_CLAUDE_PATH is not None: return ( @@ -1516,6 +1522,8 @@ def _build_claude_command(request: dict[str, Any]) -> list[str]: cmd[2:2] = ["--bare"] else: cmd[2:2] = ["--setting-sources", "project,local"] + if settings.AGENT_CLAUDE_EFFORT is not None: + cmd[2:2] = ["--effort", settings.AGENT_CLAUDE_EFFORT] if options["max_turns"] is not None: cmd[2:2] = ["--max-turns", str(options["max_turns"])] return cmd @@ -1926,14 +1934,18 @@ async def _terminate_process_group(pgid: int) -> bool: await asyncio.sleep(_AGENT_PROCESS_ABORT_GRACE_SECONDS) if not _is_process_group_running(pgid): return True - _send_process_group_signal(pgid, signal.SIGKILL) + _send_process_group_signal(pgid, _get_kill_signal()) await asyncio.sleep(0) return not _is_process_group_running(pgid) def _send_process_group_signal(pgid: int, sig: signal.Signals) -> bool: + killpg: Optional[Callable[[int, signal.Signals], None]] = getattr(os, "killpg", None) try: - os.killpg(pgid, sig) + if killpg is not None: + killpg(pgid, sig) + else: + os.kill(pgid, sig) except ProcessLookupError: return False except PermissionError: @@ -1996,8 +2008,11 @@ def _is_process_running(pid: int) -> bool: def _is_process_group_running(pgid: int) -> bool: + killpg: Optional[Callable[[int, int], None]] = getattr(os, "killpg", None) + if killpg is None: + return _is_process_running(pgid) try: - os.killpg(pgid, 0) + killpg(pgid, 0) except ProcessLookupError: return False except PermissionError: @@ -2006,6 +2021,10 @@ def _is_process_group_running(pgid: int) -> bool: return True +def _get_kill_signal() -> signal.Signals: + return getattr(signal, "SIGKILL", signal.SIGTERM) + + def _has_non_zombie_process_in_group(pgid: int) -> Optional[bool]: try: result = subprocess.run( diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index 37d95dd7d7..e0892fca0c 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -41,6 +41,7 @@ def _getenv_non_empty(name: str) -> str | None: AGENT_CLAUDE_USE_EXISTING_AUTH = environ.get_bool( "DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH", default=False ) +AGENT_CLAUDE_EFFORT = _getenv_non_empty("DSTACK_AGENT_CLAUDE_EFFORT") AGENT_ANTHROPIC_MODEL = os.getenv("DSTACK_AGENT_ANTHROPIC_MODEL", "claude-opus-4-8") DATABASE_URL = os.getenv( diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index 831a848317..bb8f684728 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -87,6 +87,11 @@ def _configure_fake_claude(tmp_path, monkeypatch: pytest.MonkeyPatch) -> str: return str(claude_path) +@pytest.fixture(autouse=True) +def _clear_claude_effort(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(settings, "AGENT_CLAUDE_EFFORT", None) + + class TestClaudeAgentService: @pytest.mark.asyncio async def test_default_runner_starts_agent_detached( @@ -513,6 +518,7 @@ async def runner(workspace, request): assert command[0] == claude_path assert "--bare" in command assert "--setting-sources" not in command + assert "--effort" not in command assert command[command.index("--tools") + 1] == captured["request"]["options"]["tools"] assert "--permission-mode" in command assert command[command.index("--permission-mode") + 1] == "bypassPermissions" @@ -615,6 +621,7 @@ async def runner(workspace, request): command = _build_claude_command(captured["request"]) assert "--bare" not in command assert command[command.index("--setting-sources") + 1] == "project,local" + assert "--effort" not in command @pytest.mark.asyncio async def test_includes_endpoint_profile_constraints_in_prompt( @@ -852,6 +859,43 @@ def test_returns_error_when_agent_key_is_blank(self, monkeypatch: pytest.MonkeyP "Claude CLI auth with DSTACK_AGENT_CLAUDE_USE_EXISTING_AUTH=1." ) + @pytest.mark.parametrize("effort", ["low", "medium", "high", "xhigh", "max"]) + def test_claude_command_includes_configured_effort( + self, tmp_path, monkeypatch: pytest.MonkeyPatch, effort: str + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", False) + monkeypatch.setattr(settings, "AGENT_CLAUDE_EFFORT", effort) + _configure_fake_claude(tmp_path, monkeypatch) + + command = _build_claude_command( + { + "prompt": "prompt", + "options": { + "allowed_tools": "Bash", + "disallowed_tools": "", + "model": "test-model", + "max_turns": None, + "json_schema": {}, + }, + } + ) + + assert get_claude_agent_unavailable_reason() is None + assert command[command.index("--effort") + 1] == effort + + def test_returns_error_when_claude_effort_is_invalid( + self, tmp_path, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + monkeypatch.setattr(settings, "AGENT_CLAUDE_USE_EXISTING_AUTH", False) + monkeypatch.setattr(settings, "AGENT_CLAUDE_EFFORT", "turbo") + _configure_fake_claude(tmp_path, monkeypatch) + + assert get_claude_agent_unavailable_reason() == ( + "DSTACK_AGENT_CLAUDE_EFFORT must be one of: low, medium, high, xhigh, max." + ) + def test_existing_claude_auth_opt_in_enables_agent_without_api_key( self, tmp_path, monkeypatch: pytest.MonkeyPatch ): From 5701f5204958865c401b470dac27ad54481eeb53 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Wed, 8 Jul 2026 22:33:51 +0200 Subject: [PATCH 20/21] Wait for stopped runs before endpoint handoff --- skills/dstack-prototyping/SKILL.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/skills/dstack-prototyping/SKILL.md b/skills/dstack-prototyping/SKILL.md index c043b1022e..516a912846 100644 --- a/skills/dstack-prototyping/SKILL.md +++ b/skills/dstack-prototyping/SKILL.md @@ -70,6 +70,12 @@ the requested model. For a chat or reasoning model, check the response behavior the endpoint is expected to support, such as reasoning output when that model is supposed to expose it. +If you stop a run to submit another one, whether it is a task or a service, +ensure the previous run has fully stopped, for example by reaching `stopped`, +`terminated`, or another terminal status. This can prevent provisioning extra +instances by reusing idle instances when possible, and can also preserve +instance volumes. + ## Verify As A Service Submit the service after the task has verified the recipe: image, command, port, From 4371b81cfd5e58b1e979c525f2fde8c8600e4e9f Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Fri, 10 Jul 2026 23:03:26 +0200 Subject: [PATCH 21/21] Add endpoint model variant support --- src/dstack/_internal/cli/commands/endpoint.py | 2 +- .../cli/services/configurators/endpoint.py | 2 +- src/dstack/_internal/cli/utils/endpoint.py | 14 +- src/dstack/_internal/cli/utils/preset.py | 27 ++- .../_internal/core/models/endpoint_presets.py | 5 +- src/dstack/_internal/core/models/endpoints.py | 90 +++++++++- .../background/pipeline_tasks/endpoints.py | 98 ++++++++++- ...00_e03d97df7c5a_add_endpoint_model_repo.py | 28 +++ src/dstack/_internal/server/models.py | 2 + .../server/services/endpoints/__init__.py | 10 +- .../server/services/endpoints/agent/claude.py | 23 ++- .../server/services/endpoints/agent/report.py | 8 + .../agent/resources/system_prompt.md | 64 ++++++- .../server/services/endpoints/planning.py | 13 +- .../services/endpoints/preset_building.py | 19 +- .../server/services/endpoints/presets.py | 125 +++++++------ .../services/configurators/test_endpoint.py | 2 +- .../_internal/cli/utils/test_endpoint.py | 14 ++ src/tests/_internal/cli/utils/test_preset.py | 25 ++- .../_internal/core/models/test_endpoints.py | 75 +++++++- .../pipeline_tasks/test_endpoints.py | 140 ++++++++++++++- .../server/routers/test_endpoints.py | 23 ++- .../services/endpoints/test_agent_report.py | 32 ++++ .../services/endpoints/test_claude_agent.py | 159 ++++++++++++----- .../server/services/test_endpoint_presets.py | 165 ++++++++++++++++-- 25 files changed, 997 insertions(+), 168 deletions(-) create mode 100644 src/dstack/_internal/server/migrations/versions/2026/07_10_1200_e03d97df7c5a_add_endpoint_model_repo.py diff --git a/src/dstack/_internal/cli/commands/endpoint.py b/src/dstack/_internal/cli/commands/endpoint.py index 17dd39b535..89635f1cb1 100644 --- a/src/dstack/_internal/cli/commands/endpoint.py +++ b/src/dstack/_internal/cli/commands/endpoint.py @@ -296,7 +296,7 @@ def _preset_get(self, args: argparse.Namespace): def _preset_delete(self, args: argparse.Namespace): presets = self.api.client.endpoint_presets.list(self.api.project) - if args.model not in {preset.model for preset in presets}: + if args.model not in {preset.base for preset in presets}: console.print(f"Endpoint preset for model [code]{args.model}[/] does not exist") exit(1) diff --git a/src/dstack/_internal/cli/services/configurators/endpoint.py b/src/dstack/_internal/cli/services/configurators/endpoint.py index 3afd8ab81a..149c06c47f 100644 --- a/src/dstack/_internal/cli/services/configurators/endpoint.py +++ b/src/dstack/_internal/cli/services/configurators/endpoint.py @@ -271,7 +271,7 @@ def th(s: str) -> str: props.add_row(th("Max price"), _format_max_price(plan)) props.add_row(th("Preset policy"), plan.preset_policy.value) if isinstance(plan.provisioning_plan, EndpointProvisioningPlanPreset): - props.add_row(th("Preset"), plan.provisioning_plan.preset_model) + props.add_row(th("Preset"), plan.provisioning_plan.preset_base) props.add_row(th("Recipe"), plan.provisioning_plan.recipe_id) console.print(props) console.print() diff --git a/src/dstack/_internal/cli/utils/endpoint.py b/src/dstack/_internal/cli/utils/endpoint.py index 5dd740d613..46f932626c 100644 --- a/src/dstack/_internal/cli/utils/endpoint.py +++ b/src/dstack/_internal/cli/utils/endpoint.py @@ -53,7 +53,10 @@ def th(value: str) -> str: table.add_row(th("Project"), endpoint.project_name) table.add_row(th("User"), endpoint.user) table.add_row(th("Endpoint"), endpoint.name) - table.add_row(th("Model"), endpoint.configuration.model) + model = endpoint.configuration.model.api_model_name + table.add_row(th("Model"), model) + if endpoint.model_repo is not None and endpoint.model_repo != model: + table.add_row(th("Repo"), endpoint.model_repo) table.add_row( th("Status"), _format_endpoint_status( @@ -88,9 +91,10 @@ def get_endpoints_table( table.add_column("ERROR") for endpoint in endpoints: + model = endpoint.configuration.model.api_model_name row = { "NAME": endpoint.name, - "MODEL": endpoint.configuration.model, + "MODEL": model, "STATUS": _format_endpoint_status( endpoint.status, endpoint.status_message, @@ -102,6 +106,12 @@ def get_endpoints_table( "ERROR": endpoint.status_message, } add_row_from_dict(table, row) + if endpoint.model_repo is not None and endpoint.model_repo != model: + add_row_from_dict( + table, + {"MODEL": f" repo={endpoint.model_repo}"}, + style="secondary", + ) return table diff --git a/src/dstack/_internal/cli/utils/preset.py b/src/dstack/_internal/cli/utils/preset.py index a5b066c6c9..a8f5a4b274 100644 --- a/src/dstack/_internal/cli/utils/preset.py +++ b/src/dstack/_internal/cli/utils/preset.py @@ -25,15 +25,17 @@ def get_endpoint_presets_table(presets: List[EndpointPreset], verbose: bool = Fa if len(preset.recipes) == 1: _add_recipe_rows( table, - label=f"[bold]{preset.model}[/]", + preset_base=preset.base, + label=f"[bold]{preset.base}[/]", recipe=preset.recipes[0], verbose=verbose, ) continue - add_row_from_dict(table, {"MODEL": f"[bold]{preset.model}[/]"}) + add_row_from_dict(table, {"MODEL": f"[bold]{preset.base}[/]"}) for recipe_num, recipe in enumerate(preset.recipes): _add_recipe_rows( table, + preset_base=preset.base, label=f"[secondary] recipe={recipe_num}[/]", recipe=recipe, verbose=verbose, @@ -43,6 +45,7 @@ def get_endpoint_presets_table(presets: List[EndpointPreset], verbose: bool = Fa def _add_recipe_rows( table: Table, + preset_base: str, label: str, recipe: EndpointPresetRecipe, verbose: bool, @@ -57,9 +60,19 @@ def _add_recipe_rows( column: _format_resources(groups[0].resources, verbose=verbose), }, ) + _add_recipe_model_row( + table, + preset_base=preset_base, + recipe_model=recipe.model, + ) return add_row_from_dict(table, {"MODEL": label}) + _add_recipe_model_row( + table, + preset_base=preset_base, + recipe_model=recipe.model, + ) for group in groups: add_row_from_dict( table, @@ -70,6 +83,16 @@ def _add_recipe_rows( ) +def _add_recipe_model_row( + table: Table, + preset_base: str, + recipe_model: str, +) -> None: + if recipe_model == preset_base: + return + add_row_from_dict(table, {"MODEL": f" repo={recipe_model}"}, style="secondary") + + def _format_resources(resources, verbose: bool) -> str: if not verbose: if resources.gpu is None: diff --git a/src/dstack/_internal/core/models/endpoint_presets.py b/src/dstack/_internal/core/models/endpoint_presets.py index 1e8b7db79b..f3ed18ff8e 100644 --- a/src/dstack/_internal/core/models/endpoint_presets.py +++ b/src/dstack/_internal/core/models/endpoint_presets.py @@ -15,10 +15,13 @@ class EndpointPresetValidation(CoreModel): class EndpointPresetRecipe(CoreModel): id: str + model: str + """Exact repo/path loaded by this recipe's service command.""" service: ServiceConfiguration validations: list[EndpointPresetValidation] class EndpointPreset(CoreModel): - model: str + base: str + """Base/requested/API model name used for preset lookup and client requests.""" recipes: list[EndpointPresetRecipe] diff --git a/src/dstack/_internal/core/models/endpoints.py b/src/dstack/_internal/core/models/endpoints.py index 032b96921c..ab95fd43c5 100644 --- a/src/dstack/_internal/core/models/endpoints.py +++ b/src/dstack/_internal/core/models/endpoints.py @@ -1,9 +1,9 @@ import uuid from datetime import datetime from enum import Enum -from typing import Annotated, Literal, Optional, Union +from typing import Annotated, Any, Literal, Optional, Union -from pydantic import Field +from pydantic import Field, validator from dstack._internal.core.models.common import ( ApplyAction, @@ -14,6 +14,7 @@ from dstack._internal.core.models.instances import InstanceOfferWithAvailability from dstack._internal.core.models.profiles import ProfileParams, ProfileParamsConfig from dstack._internal.core.models.resources import ResourcesSpec +from dstack._internal.utils.json_schema import add_extra_schema_types class EndpointStatus(str, Enum): @@ -43,6 +44,63 @@ class EndpointConfigurationConfig(ProfileParamsConfig): @staticmethod def schema_extra(schema: dict): ProfileParamsConfig.schema_extra(schema) + add_extra_schema_types( + schema["properties"]["model"], + extra_types=[{"type": "string"}], + ) + + +class EndpointModelRepo(CoreModel): + repo: str + """Exact repo/path to deploy.""" + name: Optional[str] = None + """API-facing model name. Defaults to `repo`.""" + + @property + def api_model_name(self) -> str: + return self.name or self.repo + + @property + def exact_repo(self) -> str: + return self.repo + + @property + def allows_variant_selection(self) -> bool: + return False + + @validator("repo") + def validate_repo(cls, v: str) -> str: + return _validate_endpoint_model_string(v, field_name="repo") + + @validator("name") + def validate_name(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + return _validate_endpoint_model_string(v, field_name="name") + + +class EndpointModelBase(CoreModel): + base: str + """Base model name. The agent may choose a compatible repo/path to deploy.""" + + @property + def api_model_name(self) -> str: + return self.base + + @property + def exact_repo(self) -> None: + return None + + @property + def allows_variant_selection(self) -> bool: + return True + + @validator("base") + def validate_base(cls, v: str) -> str: + return _validate_endpoint_model_string(v, field_name="base") + + +EndpointModelSpec = Union[EndpointModelRepo, EndpointModelBase] class EndpointConfiguration( @@ -54,9 +112,15 @@ class EndpointConfiguration( Optional[str], Field(description="The endpoint name. If not specified, a random name is generated"), ] = None + model: Annotated[ - str, - Field(description="The model to serve, typically a Hugging Face model ID"), + EndpointModelSpec, + Field( + description=( + "The model to serve. Use a string or `repo` for an exact repo/path, " + "or `base` to let the endpoint agent choose a compatible variant." + ) + ), ] env: Annotated[ Env, @@ -76,6 +140,12 @@ class EndpointConfiguration( ), ] = EndpointPresetPolicy.REUSE_OR_CREATE + @validator("model", pre=True) + def parse_model(cls, v) -> dict | EndpointModelSpec: + if isinstance(v, str): + return {"repo": _validate_endpoint_model_string(v, field_name="model")} + return v + class Endpoint(CoreModel): id: uuid.UUID @@ -89,6 +159,10 @@ class Endpoint(CoreModel): status_message: Optional[str] = None run_name: Optional[str] = None url: Optional[str] = None + model_base: Optional[str] = None + """Base model repo for the endpoint's current deployed model.""" + model_repo: Optional[str] = None + """Exact repo/path deployed by the endpoint's current service.""" error: Optional[str] = None @@ -109,7 +183,7 @@ class EndpointPlanJobOffers(CoreModel): class EndpointProvisioningPlanPreset(CoreModel): type: Literal["preset"] = "preset" - preset_model: str + preset_base: str recipe_id: str service_name: str job_offers: list[EndpointPlanJobOffers] @@ -137,3 +211,9 @@ class EndpointPlan(CoreModel): action: ApplyAction preset_policy: EndpointPresetPolicy provisioning_plan: Annotated[AnyEndpointProvisioningPlan, Field(discriminator="type")] + + +def _validate_endpoint_model_string(value: Any, *, field_name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"Endpoint model {field_name} must be a non-empty string") + return value diff --git a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py index 39bc45768d..3e5683c2b4 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/endpoints.py @@ -10,7 +10,11 @@ from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.configurations import ServiceConfiguration -from dstack._internal.core.models.endpoints import EndpointPresetPolicy, EndpointStatus +from dstack._internal.core.models.endpoints import ( + EndpointConfiguration, + EndpointPresetPolicy, + EndpointStatus, +) from dstack._internal.core.models.runs import ( ApplyRunPlanInput, JobStatus, @@ -355,6 +359,12 @@ async def _link_reported_service_run_to_stopping_endpoint( return False update_map = _EndpointUpdateMap(service_run_id=service_run_id) + model_repo = result.update_map.get("model_repo") + if model_repo is not None: + update_map["model_repo"] = model_repo + model_base = result.update_map.get("model_base") + if model_base is not None: + update_map["model_base"] = model_base set_processed_update_map_fields(update_map) set_unlock_update_map_fields(update_map) @@ -386,6 +396,8 @@ class _EndpointUpdateMap(ItemUpdateMap, total=False): status: EndpointStatus status_message: Optional[str] service_run_id: uuid.UUID + model_base: Optional[str] + model_repo: Optional[str] provisioning_method: Optional[str] @@ -397,8 +409,9 @@ class _ProcessResult: @dataclass(frozen=True) class _PresetSubmission: run_id: uuid.UUID - preset_model: str + preset_base: str recipe_id: str + model_repo: str @dataclass(frozen=True) @@ -476,15 +489,17 @@ async def _process_submitted_endpoint( logger.info( "Provisioning endpoint %s from preset %s recipe %s", endpoint_model.name, - submission_result.submission.preset_model, + submission_result.submission.preset_base, submission_result.submission.recipe_id, ) update_map = _EndpointUpdateMap( status=EndpointStatus.PROVISIONING, status_message=None, service_run_id=submission_result.submission.run_id, + model_base=submission_result.submission.preset_base, + model_repo=submission_result.submission.model_repo, provisioning_method=( - f"preset:{submission_result.submission.preset_model}" + f"preset:{submission_result.submission.preset_base}" f"#{submission_result.submission.recipe_id}" ), ) @@ -631,6 +646,8 @@ async def _process_agent_verified_endpoint( await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, + base_model=endpoint_model.model_base, + recipe_model=endpoint_model.model_repo, ) await _stop_non_final_submitted_runs( endpoint_model=endpoint_model, @@ -725,6 +742,17 @@ async def _provision_endpoint_with_agent( "status_message": "Server agent final report did not identify a run id", } ) + model_base, model_repo, model_error = _get_agent_report_models( + endpoint_configuration=get_endpoint_configuration(endpoint_model), + report=report, + ) + if model_error is not None: + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": model_error, + } + ) async with get_session_ctx() as session: res = await session.execute( @@ -785,6 +813,18 @@ async def _provision_endpoint_with_agent( "status_message": f"Run '{run_model.run_name}' is not a service", } ) + endpoint_configuration = get_endpoint_configuration(endpoint_model) + requested_model_name = endpoint_configuration.model.api_model_name + if ( + run_spec.configuration.model is None + or run_spec.configuration.model.name != requested_model_name + ): + return _ProcessResult( + update_map={ + "status": EndpointStatus.FAILED, + "status_message": "Server agent final service model does not match the endpoint model", + } + ) try: await _record_endpoint_run_submission( endpoint_id=endpoint_model.id, @@ -811,11 +851,15 @@ async def _provision_endpoint_with_agent( "status": EndpointStatus.PROTOTYPING, "status_message": None, "service_run_id": run_model.id, + "model_base": model_base, + "model_repo": model_repo, } ) await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, + base_model=model_base, + recipe_model=model_repo, ) await _stop_non_final_submitted_runs( endpoint_model=endpoint_model, @@ -827,10 +871,39 @@ async def _provision_endpoint_with_agent( "status": EndpointStatus.RUNNING, "status_message": None, "service_run_id": run_model.id, + "model_base": model_base, + "model_repo": model_repo, } ) +def _get_agent_report_models( + *, + endpoint_configuration: EndpointConfiguration, + report, +) -> tuple[Optional[str], Optional[str], Optional[str]]: + exact_model_repo = endpoint_configuration.model.exact_repo + if report.base is None: + return None, None, "Server agent final report did not identify the base model repo" + if endpoint_configuration.model.allows_variant_selection: + requested_base = endpoint_configuration.model.api_model_name + if report.base != requested_base: + return ( + None, + None, + "Server agent final report base does not match the requested base model", + ) + if report.model is None: + return None, None, "Server agent final report did not identify the selected model repo" + if exact_model_repo is not None and report.model != exact_model_repo: + return ( + None, + None, + "Server agent final report model does not match the requested model repo", + ) + return report.base, report.model, None + + def _format_agent_status_message( message: Optional[str], default: str = "Server agent failed", @@ -1006,15 +1079,23 @@ async def _try_save_agent_endpoint_preset( await _save_agent_endpoint_preset( endpoint_model=endpoint_model, run_model=run_model, + base_model=endpoint_model.model_base, + recipe_model=endpoint_model.model_repo, ) async def _save_agent_endpoint_preset( endpoint_model: EndpointModel, run_model: RunModel, + base_model: Optional[str] = None, + recipe_model: Optional[str] = None, ) -> None: try: - preset = build_endpoint_preset_from_run(run_model) + preset = build_endpoint_preset_from_run( + run_model, + base_model=base_model, + recipe_model=recipe_model, + ) saved_preset = await get_endpoint_preset_service().save_preset( endpoint_model.project.name, preset, @@ -1036,7 +1117,7 @@ async def _save_agent_endpoint_preset( recipe_ids = ", ".join(recipe.id for recipe in saved_preset.recipes) logger.info( "Saved endpoint preset for model %s recipes %s for endpoint %s", - saved_preset.model, + saved_preset.base, recipe_ids, endpoint_model.name, ) @@ -1153,8 +1234,9 @@ async def _submit_endpoint_from_preset( return _PresetSubmissionResult( submission=_PresetSubmission( run_id=run.id, - preset_model=preset_plan.preset.model, + preset_base=preset_plan.preset.base, recipe_id=preset_plan.recipe.id, + model_repo=preset_plan.recipe.model, ) ) @@ -1224,7 +1306,7 @@ def _get_no_provisioning_path_message( def _format_preset_plan_label(preset_plan) -> str: - return f"for model {preset_plan.preset.model} recipe {preset_plan.recipe.id}" + return f"for model {preset_plan.preset.base} recipe {preset_plan.recipe.id}" async def _stop_backing_run( diff --git a/src/dstack/_internal/server/migrations/versions/2026/07_10_1200_e03d97df7c5a_add_endpoint_model_repo.py b/src/dstack/_internal/server/migrations/versions/2026/07_10_1200_e03d97df7c5a_add_endpoint_model_repo.py new file mode 100644 index 0000000000..35e36d7e26 --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/07_10_1200_e03d97df7c5a_add_endpoint_model_repo.py @@ -0,0 +1,28 @@ +"""add endpoint model fields + +Revision ID: e03d97df7c5a +Revises: 4e1d6c2a9b77 +Create Date: 2026-07-10 12:00:00.000000+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e03d97df7c5a" +down_revision = "4e1d6c2a9b77" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("endpoints", schema=None) as batch_op: + batch_op.add_column(sa.Column("model_base", sa.Text(), nullable=True)) + batch_op.add_column(sa.Column("model_repo", sa.Text(), nullable=True)) + + +def downgrade() -> None: + with op.batch_alter_table("endpoints", schema=None) as batch_op: + batch_op.drop_column("model_repo") + batch_op.drop_column("model_base") diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 0580abace4..6c403b15f0 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -1031,6 +1031,8 @@ class EndpointModel(PipelineModelMixin, BaseModel): status: Mapped[EndpointStatus] = mapped_column(EnumAsString(EndpointStatus, 100), index=True) """`status` must be changed only via `switch_endpoint_status()`.""" status_message: Mapped[Optional[str]] = mapped_column(Text) + model_base: Mapped[Optional[str]] = mapped_column(Text) + model_repo: Mapped[Optional[str]] = mapped_column(Text) provisioning_method: Mapped[Optional[str]] = mapped_column(String(100)) created_at: Mapped[datetime] = mapped_column(NaiveDateTime, default=get_current_datetime) diff --git a/src/dstack/_internal/server/services/endpoints/__init__.py b/src/dstack/_internal/server/services/endpoints/__init__.py index b6ea63cf54..6ff1912646 100644 --- a/src/dstack/_internal/server/services/endpoints/__init__.py +++ b/src/dstack/_internal/server/services/endpoints/__init__.py @@ -479,7 +479,7 @@ def _get_unprovisionable_preset_reason( if preset_plan is None: return None return ( - f"Endpoint preset for model {preset_plan.preset.model} " + f"Endpoint preset for model {preset_plan.preset.base} " f"recipe {preset_plan.recipe.id} matched but has no available offers." ) @@ -490,7 +490,7 @@ def _endpoint_preset_plan_to_provisioning_plan( run_spec = preset_plan.run_plan.get_effective_run_spec() service_name = run_spec.run_name or run_spec.configuration.name or "(generated)" return EndpointProvisioningPlanPreset( - preset_model=preset_plan.preset.model, + preset_base=preset_plan.preset.base, recipe_id=preset_plan.recipe.id, service_name=service_name, job_offers=[ @@ -545,6 +545,8 @@ async def create_endpoint( endpoint_model.user = user endpoint_model.service_run_id = None endpoint_model.service_run = None + endpoint_model.model_base = None + endpoint_model.model_repo = None endpoint_model.configuration = configuration.json() endpoint_model.status = EndpointStatus.SUBMITTED endpoint_model.status_message = None @@ -685,6 +687,8 @@ def endpoint_model_to_endpoint(endpoint_model: EndpointModel) -> Endpoint: status_message=endpoint_model.status_message, run_name=run_name, url=url, + model_base=endpoint_model.model_base, + model_repo=endpoint_model.model_repo, error=( endpoint_model.status_message if endpoint_model.status == EndpointStatus.FAILED @@ -752,7 +756,7 @@ async def generate_endpoint_name(session: AsyncSession, project: ProjectModel) - def _validate_endpoint_configuration(configuration: EndpointConfiguration): - if not configuration.model.strip(): + if not configuration.model.api_model_name.strip(): raise ServerClientError("Endpoint must specify model") if configuration.name is not None: validate_dstack_resource_name(configuration.name) diff --git a/src/dstack/_internal/server/services/endpoints/agent/claude.py b/src/dstack/_internal/server/services/endpoints/agent/claude.py index 134cd44667..a84f14d32d 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/claude.py +++ b/src/dstack/_internal/server/services/endpoints/agent/claude.py @@ -767,6 +767,7 @@ def __init__( redacted_values: Sequence[str], endpoint_name: str, model: str, + endpoint_context: Optional[str] = None, dstack_home_dir: Optional[Path] = None, endpoint_constraints: str = "", max_price: Optional[float] = None, @@ -783,6 +784,9 @@ def __init__( self.redacted_values = tuple(redacted_values) self.endpoint_name = endpoint_name self.model = model + self.endpoint_context = endpoint_context or ( + f"- service_model_name: {model}\n- model_repo: {model}" + ) self.endpoint_constraints = endpoint_constraints self.max_price = max_price self.spot_policy = spot_policy @@ -890,6 +894,16 @@ def _prepare_workspace( ] ) trace_path = root_dir / "trace.jsonl" if _is_debug_trace_enabled() else None + if configuration.model.allows_variant_selection: + endpoint_context = ( + f"- service_model_name: {configuration.model.api_model_name}\n" + f"- base_model: {configuration.model.api_model_name}" + ) + else: + endpoint_context = ( + f"- service_model_name: {configuration.model.api_model_name}\n" + f"- model_repo: {configuration.model.exact_repo}" + ) workspace = _AgentWorkspace( root_dir=root_dir, home_dir=home_dir, @@ -899,7 +913,8 @@ def _prepare_workspace( env=env, redacted_values=redacted_values, endpoint_name=endpoint_model.name, - model=configuration.model, + model=configuration.model.api_model_name, + endpoint_context=endpoint_context, endpoint_constraints=agent_constraints.prompt_text, max_price=configuration.max_price, spot_policy=configuration.spot_policy.value if configuration.spot_policy else None, @@ -1321,9 +1336,9 @@ def _build_agent_request(workspace: _AgentWorkspace) -> dict[str, Any]: def _build_prompt(workspace: _AgentWorkspace) -> str: return f"""{_load_endpoint_prompt()} -Endpoint request: -- name: {workspace.endpoint_name} -- model: {workspace.model} +Endpoint context: +- endpoint_name: {workspace.endpoint_name} +{workspace.endpoint_context} {workspace.endpoint_constraints} """ diff --git a/src/dstack/_internal/server/services/endpoints/agent/report.py b/src/dstack/_internal/server/services/endpoints/agent/report.py index 8fb10b8d08..a601869b79 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/report.py +++ b/src/dstack/_internal/server/services/endpoints/agent/report.py @@ -12,6 +12,8 @@ "run_id": {"type": "string"}, "run_name": {"type": "string"}, "service_yaml": {"type": "string"}, + "base": {"type": "string"}, + "model": {"type": "string"}, "failure_summary": {"type": "string"}, }, "required": ["success"], @@ -26,6 +28,8 @@ class AgentFinalReport(CoreModel): run_id: Optional[uuid.UUID] = None run_name: Optional[str] = None service_yaml: Optional[str] = None + base: Optional[str] = None + model: Optional[str] = None failure_summary: Optional[str] = None @root_validator @@ -36,6 +40,10 @@ def _validate_report(cls, values: dict) -> dict: raise ValueError("successful agent report must include run_id") if not values.get("service_yaml"): raise ValueError("successful agent report must include service_yaml") + if not values.get("base") or not values["base"].strip(): + raise ValueError("successful agent report must include base") + if not values.get("model") or not values["model"].strip(): + raise ValueError("successful agent report must include model") else: if not values.get("failure_summary"): raise ValueError("failed agent report must include failure_summary") diff --git a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md index 69992108e4..3c74417d31 100644 --- a/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md +++ b/src/dstack/_internal/server/services/endpoints/agent/resources/system_prompt.md @@ -1,8 +1,19 @@ # Objective You are the endpoint deployment agent for dstack. Produce one final dstack -service for the requested model. Report success only after that service answers -a real request for the requested model through the dstack service URL. +service. Report success only after that service answers a real request using +`service_model_name` from `Endpoint context:` through the dstack service URL. + +# Requested Model + +The `Endpoint context:` block contains either `model_repo` or `base_model`. + +- If it contains `model_repo`, deploy that repo/path exactly. +- If it contains `base_model`, choose a repo/path compatible with `base_model` + that best fits performance and hardware within the endpoint constraints, + allowed fleets, backends, and offers. A variant can be the base repo itself, a + different precision or quantization, or another trusted repo compatible with + `base_model`. Use the real `dstack` CLI and shell commands in this workspace. Load and follow `/dstack` for dstack CLI/YAML syntax. Load and follow `/dstack-prototyping` for @@ -130,6 +141,19 @@ selecting fleet, backend, and hardware. Choosing fleet/backend is gated by classifying each against `https://dstack.ai/docs/concepts/backends.md`. VM-based backends are listed under `## VM-based` (they support idle instances and instance volumes). Kubernetes backend is listed under `## Container-based`, but supports instance volumes and thus is preferred over other container-based backends. SSH fleets can be treated as VM-based backends as they support both idle instances (its equivalent) and instance volumes. +## Model And Compute Fit + +If `Endpoint context:` contains `model_repo`, choose fleet/backend/hardware that +can run `model_repo` within endpoint constraints. + +If `Endpoint context:` contains `base_model`, choose the repo/path and compute +together. You may pick the base repo or a compatible variant that fits the +allowed fleets, backends, and offers. + +If a task or service shows that the selected repo/path is a bad fit and +`Endpoint context:` contains `base_model`, pick another compatible variant if +available and test it in a task before submitting another service. + ## Submitting run When submitting a task or service, pass exact `fleets`, `backends`, and an @@ -140,15 +164,18 @@ does not land outside the intended fleet/backend/hardware. Endpoint logs should explain meaningful decisions and actions. -When choosing fleet, backend, and hardware, write a log message that includes: +When choosing repo/path for `base_model`, fleet, backend, or hardware, write a +log message that includes: +- `service_model_name` and selected repo/path, when they differ; - the selected fleet, backend, and resource range; - the offer/docs evidence used for the choice; - the viable alternatives not selected and the exact reason; - the fleet, backend, and resources that will be used in the submitted YAML. - how the selected and rejected fleets/backends were classified; -Backend-choice log example: +Backend-choice log example (provide the same level of explanation for repo/path, +fleet, and hardware choice): "I chose backend ... on fleet ... because ...; I did not choose ... because ...; I will submit YAML with fleets=..., backends=..., resources=...." @@ -157,8 +184,14 @@ I will submit YAML with fleets=..., backends=..., resources=...." The final `service_yaml` is what dstack will save as the endpoint recipe. It must contain the full service config: `type: service`, the final run name, the -requested model, image/commands/port, resources, env references, and the fixed -endpoint constraints that apply to service YAML. +service model name, image/commands/port, resources, env references, and the +fixed endpoint constraints that apply to service YAML. + +Set final `service_yaml.name` to the final service run name. +Set the final service model name to `service_model_name` from `Endpoint +context:`. If final `service_yaml.model` is a string, set it to +`service_model_name`; if it is an object, set `model.name` to +`service_model_name`. Choose service resources from the least restrictive requirements supported by the evidence, not from the exact machine that happened to run. For example, if @@ -172,7 +205,8 @@ failed. Use logs to understand failures. When dstack exposes the service URL, send a real model request through that URL. Do not write a successful `final_report.json` until the final service answers a -request for the requested model through the dstack service URL. +request using `service_model_name` from `Endpoint context:` through the dstack +service URL. # Secrets @@ -192,12 +226,26 @@ the next run number and record why the old run is not enough. # Final Report On success, write `final_report.json`, then return the structured final report. +The successful report must include: + +- `base`: the base model repo for the deployed model +- `model`: the exact repo/path loaded by the final service command + +Set `model` to the exact repo/path loaded by the service command. + +Set `base` as follows: + +- If `Endpoint context:` contains `base_model`, set `base` to `base_model`. +- If `Endpoint context:` contains `model_repo`, inspect the repo metadata, model + card, config, or another reliable source to identify the base model repo. +- If `model_repo` is itself the base model repo, set `base` to `model_repo`. +- Do not infer `base` only from the repo name. On failure, write `final_report.json` with a useful `failure_summary`, then return the structured final report. `final_report.json` must contain only the schema fields: `success`, `run_id`, -`run_name`, `service_yaml`, and `failure_summary`. +`run_name`, `service_yaml`, `base`, `model`, and `failure_summary`. Stop after one correct service is verified. P/D disaggregation is not covered by the current endpoint agent or `/dstack-prototyping` skill. diff --git a/src/dstack/_internal/server/services/endpoints/planning.py b/src/dstack/_internal/server/services/endpoints/planning.py index b40294bfdb..2602ca92dc 100644 --- a/src/dstack/_internal/server/services/endpoints/planning.py +++ b/src/dstack/_internal/server/services/endpoints/planning.py @@ -5,7 +5,9 @@ from dstack._internal.core.errors import ServerClientError from dstack._internal.core.models.configurations import ServiceConfiguration -from dstack._internal.core.models.endpoints import EndpointConfiguration +from dstack._internal.core.models.endpoints import ( + EndpointConfiguration, +) from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.profiles import ProfileParams from dstack._internal.core.models.runs import JobPlan, RunPlan, RunSpec @@ -74,19 +76,22 @@ async def find_preset_planning_result( if preset_service is None: preset_service = get_endpoint_preset_service() - endpoint_model = endpoint_configuration.model.lower() + endpoint_model = endpoint_configuration.model.api_model_name.lower() presets = [ preset for preset in await preset_service.list_presets(project.name) - if preset.model.lower() == endpoint_model + if preset.base.lower() == endpoint_model ] if not presets: return EndpointPresetPlanningResult() + exact_model_repo = endpoint_configuration.model.exact_repo user = await _ensure_user_has_ssh_key(session=session, user=user) first_unprovisionable_preset: Optional[EndpointPresetPlan] = None for preset in presets: for recipe in preset.recipes: + if exact_model_repo is not None and recipe.model != exact_model_repo: + continue try: run_spec = build_preset_run_spec( endpoint_name=endpoint_name, @@ -104,7 +109,7 @@ async def find_preset_planning_result( except (ServerClientError, ValueError) as e: logger.warning( "Skipping endpoint preset %s recipe %s: %s", - preset.model, + preset.base, recipe.id, e, ) diff --git a/src/dstack/_internal/server/services/endpoints/preset_building.py b/src/dstack/_internal/server/services/endpoints/preset_building.py index b8c9be91fa..3d2d2054e5 100644 --- a/src/dstack/_internal/server/services/endpoints/preset_building.py +++ b/src/dstack/_internal/server/services/endpoints/preset_building.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any +from typing import Any, Optional from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.instances import InstanceOfferWithAvailability, Resources @@ -19,13 +19,25 @@ from dstack._internal.utils.common import format_mib_as_gb -def build_endpoint_preset_from_run(run_model: RunModel) -> EndpointPreset: +def build_endpoint_preset_from_run( + run_model: RunModel, + base_model: Optional[str] = None, + recipe_model: Optional[str] = None, +) -> EndpointPreset: run_spec = runs_services.get_run_spec(run_model) configuration = run_spec.configuration if not isinstance(configuration, ServiceConfiguration): raise ValueError("endpoint preset can only be built from a service run") if configuration.model is None: raise ValueError("endpoint preset service must specify model") + if base_model is None: + base_model = configuration.model.name + if not base_model.strip(): + raise ValueError("endpoint preset base must be non-empty") + if recipe_model is None: + recipe_model = configuration.model.name + if not recipe_model.strip(): + raise ValueError("endpoint preset recipe must specify model") jobs_by_group = _get_current_registered_replica_jobs_by_group(run_model) validation_replicas = [] @@ -53,10 +65,11 @@ def build_endpoint_preset_from_run(run_model: RunModel) -> EndpointPreset: validations=[validation], ) return EndpointPreset( - model=configuration.model.name, + base=base_model, recipes=[ EndpointPresetRecipe( id=make_endpoint_preset_recipe_id(service), + model=recipe_model, service=service, validations=[validation], ) diff --git a/src/dstack/_internal/server/services/endpoints/presets.py b/src/dstack/_internal/server/services/endpoints/presets.py index 6a73d8023d..9361485d9b 100644 --- a/src/dstack/_internal/server/services/endpoints/presets.py +++ b/src/dstack/_internal/server/services/endpoints/presets.py @@ -39,11 +39,11 @@ async def list_presets(self, project_name: str) -> list[EndpointPreset]: pass @abstractmethod - async def get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: + async def get_preset(self, project_name: str, base: str) -> Optional[EndpointPreset]: pass @abstractmethod - async def delete_preset(self, project_name: str, model: str) -> None: + async def delete_preset(self, project_name: str, base: str) -> None: pass @abstractmethod @@ -64,11 +64,11 @@ def __init__(self, projects_dir: Path = settings.SERVER_PROJECTS_DIR_PATH) -> No async def list_presets(self, project_name: str) -> list[EndpointPreset]: return await run_async(self._list_presets, project_name) - async def get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: - return await run_async(self._get_preset, project_name, model) + async def get_preset(self, project_name: str, base: str) -> Optional[EndpointPreset]: + return await run_async(self._get_preset, project_name, base) - async def delete_preset(self, project_name: str, model: str) -> None: - return await run_async(self._delete_preset, project_name, model) + async def delete_preset(self, project_name: str, base: str) -> None: + return await run_async(self._delete_preset, project_name, base) async def save_preset( self, @@ -81,14 +81,14 @@ async def save_preset( def _list_presets(self, project_name: str) -> list[EndpointPreset]: return [preset for preset, _ in self._list_merged_presets(project_name)] - def _get_preset(self, project_name: str, model: str) -> Optional[EndpointPreset]: - preset, _ = self._find_preset_by_model(project_name, model) + def _get_preset(self, project_name: str, base: str) -> Optional[EndpointPreset]: + preset, _ = self._find_preset_by_base(project_name, base) return preset - def _delete_preset(self, project_name: str, model: str) -> None: - _, paths = self._find_preset_by_model(project_name, model) + def _delete_preset(self, project_name: str, base: str) -> None: + _, paths = self._find_preset_by_base(project_name, base) if not paths: - raise FileNotFoundError(model) + raise FileNotFoundError(base) for path in paths: path.unlink() @@ -102,13 +102,13 @@ def _save_preset( presets_dir = self._get_project_presets_dir(project_name) presets_dir.mkdir(parents=True, exist_ok=True) - existing_preset, existing_paths = self._find_preset_by_model(project_name, preset.model) + existing_preset, existing_paths = self._find_preset_by_base(project_name, preset.base) if existing_preset is not None: saved_preset = _merge_presets(existing_preset, preset, prefer_incoming=True) path = existing_paths[0] else: saved_preset = preset - path = self._get_available_preset_path(presets_dir, preset.model) + path = self._get_available_preset_path(presets_dir, preset.base) data = _preset_to_data(saved_preset) content = _format_preset_comments(comments) + yaml.safe_dump(data, sort_keys=False) @@ -120,14 +120,14 @@ def _save_preset( raise ValueError(f"saved endpoint preset {path} could not be loaded") return loaded_preset - def _find_preset_by_model( + def _find_preset_by_base( self, project_name: str, - model: str, + base: str, ) -> tuple[Optional[EndpointPreset], list[Path]]: - model_key = model.lower() + base_key = base.lower() for preset, paths in self._list_merged_presets(project_name): - if preset.model.lower() == model_key: + if preset.base.lower() == base_key: return preset, paths return None, [] @@ -136,27 +136,27 @@ def _list_merged_presets(self, project_name: str) -> list[tuple[EndpointPreset, if not presets_dir.exists(): return [] - presets_by_model: dict[str, EndpointPreset] = {} - paths_by_model: dict[str, list[Path]] = {} - model_order: list[str] = [] + presets_by_base: dict[str, EndpointPreset] = {} + paths_by_base: dict[str, list[Path]] = {} + base_order: list[str] = [] for path in self._iter_preset_paths(presets_dir): preset = self._load_preset(path) if preset is None: continue - model_key = preset.model.lower() - existing = presets_by_model.get(model_key) + base_key = preset.base.lower() + existing = presets_by_base.get(base_key) if existing is None: - presets_by_model[model_key] = preset - paths_by_model[model_key] = [path] - model_order.append(model_key) + presets_by_base[base_key] = preset + paths_by_base[base_key] = [path] + base_order.append(base_key) continue try: - presets_by_model[model_key] = _merge_presets(existing, preset) + presets_by_base[base_key] = _merge_presets(existing, preset) except ValueError as e: logger.warning("Skipping endpoint preset %s: %s", path, e) continue - paths_by_model[model_key].append(path) - return [(presets_by_model[model], paths_by_model[model]) for model in model_order] + paths_by_base[base_key].append(path) + return [(presets_by_base[base], paths_by_base[base]) for base in base_order] def _iter_preset_paths(self, presets_dir: Path) -> list[Path]: return [path for path in sorted(presets_dir.iterdir()) if path.suffix in [".yml", ".yaml"]] @@ -220,45 +220,56 @@ def _preset_from_data(data: Any) -> EndpointPreset: raise ValueError("preset must be a YAML object") if data.get("type") != "endpoint-preset": raise ValueError("preset must be an endpoint preset") - model = data.get("model") - if not isinstance(model, str) or model == "": - raise ValueError("preset must specify a model") + base = data.get("base", data.get("model")) + if not isinstance(base, str) or base == "": + raise ValueError("preset must specify a base") + legacy_model = data.get("model") + if isinstance(legacy_model, str) and "base" in data and legacy_model != base: + raise ValueError("preset base must match legacy model") if "recipes" in data: return EndpointPreset( - model=model, - recipes=_get_preset_recipes(data=data, model=model), + base=base, + recipes=_get_preset_recipes(data=data, base=base), ) return EndpointPreset( - model=model, - recipes=[_get_legacy_preset_recipe(data=data, model=model)], + base=base, + recipes=[_get_legacy_preset_recipe(data=data, base=base)], ) -def _get_preset_recipes(data: dict[str, Any], model: str) -> list[EndpointPresetRecipe]: +def _get_preset_recipes(data: dict[str, Any], base: str) -> list[EndpointPresetRecipe]: raw_recipes = data.get("recipes") if not isinstance(raw_recipes, list) or not raw_recipes: raise ValueError("preset must specify non-empty recipes") - recipes = [_get_preset_recipe(raw_recipe, model=model) for raw_recipe in raw_recipes] + recipes = [_get_preset_recipe(raw_recipe, base=base) for raw_recipe in raw_recipes] recipe_ids = [recipe.id for recipe in recipes] if len(recipe_ids) != len(set(recipe_ids)): raise ValueError("preset recipes must have unique ids") return recipes -def _get_preset_recipe(raw_recipe: Any, model: str) -> EndpointPresetRecipe: +def _get_preset_recipe(raw_recipe: Any, base: str) -> EndpointPresetRecipe: if not isinstance(raw_recipe, dict): raise ValueError("preset recipes must be objects") recipe_id = raw_recipe.get("id") if not isinstance(recipe_id, str) or recipe_id == "": raise ValueError("preset recipe must specify id") - service_data = _get_service_data(raw_recipe, model=model) + recipe_model = raw_recipe.get("model", base) + if not isinstance(recipe_model, str) or not recipe_model.strip(): + raise ValueError("preset recipe must specify model") + service_data = _get_service_data(raw_recipe, base=base) service = ServiceConfiguration.parse_obj(service_data) validations = _get_validations(raw_recipe, service=service) - return EndpointPresetRecipe(id=recipe_id, service=service, validations=validations) + return EndpointPresetRecipe( + id=recipe_id, + model=recipe_model, + service=service, + validations=validations, + ) -def _get_legacy_preset_recipe(data: dict[str, Any], model: str) -> EndpointPresetRecipe: - service_data = _get_service_data(data, model=model, allow_missing_resources=True) +def _get_legacy_preset_recipe(data: dict[str, Any], base: str) -> EndpointPresetRecipe: + service_data = _get_service_data(data, base=base, allow_missing_resources=True) raw_groups = _get_legacy_replica_spec_groups(data=data, service_data=service_data) _apply_legacy_replica_group_resources(service_data=service_data, groups=raw_groups) service = ServiceConfiguration.parse_obj(service_data) @@ -275,6 +286,7 @@ def _get_legacy_preset_recipe(data: dict[str, Any], model: str) -> EndpointPrese ) return EndpointPresetRecipe( id=make_endpoint_preset_recipe_id(service), + model=base, service=service, validations=[validation], ) @@ -282,7 +294,7 @@ def _get_legacy_preset_recipe(data: dict[str, Any], model: str) -> EndpointPrese def _get_service_data( data: dict[str, Any], - model: str, + base: str, allow_missing_resources: bool = False, ) -> dict[str, Any]: service_data = data.get("service") @@ -300,13 +312,13 @@ def _get_service_data( ) if not allow_missing_resources: _validate_service_has_resources(service_data) - service_data.setdefault("model", model) + service_data.setdefault("model", base) service_data["type"] = "service" configuration = ServiceConfiguration.parse_obj(service_data) if configuration.model is None: raise ValueError("preset service configuration must specify model") - if configuration.model.name.lower() != model.lower(): - raise ValueError("preset model must match the service model") + if configuration.model.name.lower() != base.lower(): + raise ValueError("preset base must match the service model") return service_data @@ -344,10 +356,12 @@ def _validate_preset(preset: EndpointPreset) -> None: if len(recipe_ids) != len(set(recipe_ids)): raise ValueError("preset recipes must have unique ids") for recipe in preset.recipes: + if not recipe.model.strip(): + raise ValueError("preset recipe must specify model") if recipe.service.model is None: raise ValueError("preset recipe service must specify model") - if recipe.service.model.name.lower() != preset.model.lower(): - raise ValueError("preset model must match the recipe service model") + if recipe.service.model.name.lower() != preset.base.lower(): + raise ValueError("preset base must match the recipe service model") _validate_validations(validations=recipe.validations, service=recipe.service) _validate_service_gpu_vendor_matches_validations( service=recipe.service, @@ -489,8 +503,8 @@ def _merge_presets( incoming: EndpointPreset, prefer_incoming: bool = False, ) -> EndpointPreset: - if existing.model.lower() != incoming.model.lower(): - raise ValueError("cannot merge endpoint presets for different models") + if existing.base.lower() != incoming.base.lower(): + raise ValueError("cannot merge endpoint presets for different bases") recipes = [EndpointPresetRecipe.parse_obj(recipe.dict()) for recipe in existing.recipes] recipes_by_id = {recipe.id: recipe for recipe in recipes} incoming_recipe_ids = [] @@ -502,8 +516,10 @@ def _merge_presets( recipes_by_id[recipe.id] = recipe incoming_recipe_ids.append(recipe.id) continue - if _service_configuration_to_preset_data(existing_recipe.service) != ( - _service_configuration_to_preset_data(incoming_recipe.service) + if ( + _service_configuration_to_preset_data(existing_recipe.service) + != (_service_configuration_to_preset_data(incoming_recipe.service)) + or existing_recipe.model != incoming_recipe.model ): raise ValueError(f"endpoint preset recipe id conflict: {incoming_recipe.id}") _merge_recipe_validations(existing_recipe, incoming_recipe.validations) @@ -513,7 +529,7 @@ def _merge_presets( recipes = [recipes_by_id[recipe_id] for recipe_id in incoming_recipe_ids] + [ recipe for recipe in recipes if recipe.id not in preferred_ids ] - return EndpointPreset(model=existing.model, recipes=recipes) + return EndpointPreset(base=existing.base, recipes=recipes) def _merge_recipe_validations( @@ -537,7 +553,7 @@ def _validation_key(validation: EndpointPresetValidation) -> str: def _preset_to_data(preset: EndpointPreset) -> dict[str, Any]: return { "type": "endpoint-preset", - "model": preset.model, + "base": preset.base, "recipes": [_recipe_to_data(recipe) for recipe in preset.recipes], } @@ -545,6 +561,7 @@ def _preset_to_data(preset: EndpointPreset) -> dict[str, Any]: def _recipe_to_data(recipe: EndpointPresetRecipe) -> dict[str, Any]: return { "id": recipe.id, + "model": recipe.model, "service": _service_configuration_to_preset_data(recipe.service), "validations": [ json.loads(validation.json(exclude_none=True)) for validation in recipe.validations diff --git a/src/tests/_internal/cli/services/configurators/test_endpoint.py b/src/tests/_internal/cli/services/configurators/test_endpoint.py index 1b696f1944..edb057741c 100644 --- a/src/tests/_internal/cli/services/configurators/test_endpoint.py +++ b/src/tests/_internal/cli/services/configurators/test_endpoint.py @@ -118,7 +118,7 @@ def test_prints_preset_without_resources_property(self, monkeypatch: pytest.Monk resources = ResourcesSpec.parse_obj({"gpu": "16GB", "disk": "60GB"}) plan = _get_endpoint_plan( EndpointProvisioningPlanPreset( - preset_model="Qwen/Qwen3-0.6B", + preset_base="Qwen/Qwen3-0.6B", recipe_id="vllm-a40", service_name="qwen-endpoint-serving", job_offers=[ diff --git a/src/tests/_internal/cli/utils/test_endpoint.py b/src/tests/_internal/cli/utils/test_endpoint.py index 05e089254b..850854d64e 100644 --- a/src/tests/_internal/cli/utils/test_endpoint.py +++ b/src/tests/_internal/cli/utils/test_endpoint.py @@ -18,6 +18,7 @@ def _get_endpoint( status: EndpointStatus = EndpointStatus.FAILED, status_message: str | None = "No matching endpoint presets found.", created_at: datetime | None = None, + model_repo: str | None = None, ) -> Endpoint: if created_at is None: created_at = datetime.now(timezone.utc) @@ -31,6 +32,7 @@ def _get_endpoint( last_processed_at=created_at, status=status, status_message=status_message, + model_repo=model_repo, ) @@ -127,11 +129,21 @@ def test_stopped_status_is_colored(self): status_column = next(column for column in table.columns if column.header == "STATUS") assert status_column._cells == ["[grey62]stopped[/]"] + def test_shows_repo_row_when_repo_differs_from_model(self): + table = get_endpoints_table([_get_endpoint(model_repo="groxaxo/Qwen3-0.6B-GPTQ-4Bit")]) + + model_column = next(column for column in table.columns if column.header == "MODEL") + assert model_column._cells == [ + "Qwen/Qwen3-0.6B", + " repo=groxaxo/Qwen3-0.6B-GPTQ-4Bit", + ] + class TestGetEndpointTable: def test_shows_endpoint_details(self): endpoint = _get_endpoint( status_message="No matching endpoint presets found.", + model_repo="groxaxo/Qwen3-0.6B-GPTQ-4Bit", ) table = get_endpoint_table(endpoint, format_date=lambda _: "now") @@ -141,6 +153,7 @@ def test_shows_endpoint_details(self): "[bold]User[/bold]", "[bold]Endpoint[/bold]", "[bold]Model[/bold]", + "[bold]Repo[/bold]", "[bold]Status[/bold]", "[bold]Preset policy[/bold]", "[bold]Service run[/bold]", @@ -153,6 +166,7 @@ def test_shows_endpoint_details(self): "test-user", "qwen-endpoint", "Qwen/Qwen3-0.6B", + "groxaxo/Qwen3-0.6B-GPTQ-4Bit", "[indian_red1]no preset[/]", "reuse-or-create", "-", diff --git a/src/tests/_internal/cli/utils/test_preset.py b/src/tests/_internal/cli/utils/test_preset.py index 361cc33eb5..dde551f0e1 100644 --- a/src/tests/_internal/cli/utils/test_preset.py +++ b/src/tests/_internal/cli/utils/test_preset.py @@ -39,6 +39,25 @@ def test_verbose_shows_full_resources(self): "cpu=2.. mem=8GB.. disk=100GB.. gpu=nvidia:16GB:1..", ] + def test_shows_recipe_model_when_it_differs_from_preset_base(self): + preset = _endpoint_preset( + model="Qwen/Qwen3-0.6B", + recipe_id="vllm-a40", + recipe_model="groxaxo/Qwen3-0.6B-GPTQ-4Bit", + service={"resources": {"gpu": "nvidia:16GB"}}, + ) + + table = get_endpoint_presets_table([preset]) + + assert table.columns[0]._cells == [ + "[bold]Qwen/Qwen3-0.6B[/]", + " repo=groxaxo/Qwen3-0.6B-GPTQ-4Bit", + ] + assert table.columns[1]._cells == [ + "nvidia:16GB:1..", + "", + ] + def test_shows_single_implicit_group_once_even_with_multiple_tested_replicas(self): preset = _endpoint_preset( model="Qwen/Qwen3-0.6B", @@ -98,7 +117,7 @@ def test_shows_grouped_replica_specs(self): def test_groups_multiple_recipes_by_model(self): preset = EndpointPreset( - model="Qwen/Qwen3-0.6B", + base="Qwen/Qwen3-0.6B", recipes=[ _endpoint_preset( model="Qwen/Qwen3-0.6B", @@ -133,6 +152,7 @@ def _endpoint_preset( recipe_id: str, service: dict, validation_replicas: list[dict] | None = None, + recipe_model: str | None = None, ) -> EndpointPreset: service_data = { "type": "service", @@ -143,10 +163,11 @@ def _endpoint_preset( if "replicas" not in service_data: service_data["commands"] = [f"vllm serve {model}"] return EndpointPreset( - model=model, + base=model, recipes=[ EndpointPresetRecipe( id=recipe_id, + model=recipe_model or model, service=ServiceConfiguration.parse_obj(service_data), validations=[ EndpointPresetValidation( diff --git a/src/tests/_internal/core/models/test_endpoints.py b/src/tests/_internal/core/models/test_endpoints.py index ba562a8aa7..19f033d68f 100644 --- a/src/tests/_internal/core/models/test_endpoints.py +++ b/src/tests/_internal/core/models/test_endpoints.py @@ -2,7 +2,12 @@ from dstack._internal.core.errors import ConfigurationError from dstack._internal.core.models.configurations import parse_apply_configuration -from dstack._internal.core.models.endpoints import EndpointConfiguration, EndpointPresetPolicy +from dstack._internal.core.models.endpoints import ( + EndpointConfiguration, + EndpointModelBase, + EndpointModelRepo, + EndpointPresetPolicy, +) from dstack._internal.core.models.profiles import CreationPolicy @@ -22,7 +27,8 @@ def test_parses_endpoint_configuration(self): assert isinstance(conf, EndpointConfiguration) assert conf.name == "qwen-endpoint" - assert conf.model == "Qwen/Qwen3-0.6B" + assert isinstance(conf.model, EndpointModelRepo) + assert conf.model.repo == "Qwen/Qwen3-0.6B" assert conf.env.as_dict() == {"HF_TOKEN": "secret"} assert conf.fleets is not None fleet = conf.fleets[0] @@ -36,6 +42,71 @@ def test_defaults_to_reuse_or_create_preset_policy(self): assert conf.preset_policy == EndpointPresetPolicy.REUSE_OR_CREATE + def test_parses_exact_repo_model(self): + conf = parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": {"repo": "groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit"}, + } + ) + + assert isinstance(conf.model, EndpointModelRepo) + assert conf.model.api_model_name == "groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit" + assert conf.model.exact_repo == "groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit" + assert not conf.model.allows_variant_selection + + def test_parses_exact_repo_with_api_model_name(self): + conf = parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": { + "repo": "groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit", + "name": "Qwen/Qwen3.6-27B", + }, + } + ) + + assert isinstance(conf.model, EndpointModelRepo) + assert conf.model.api_model_name == "Qwen/Qwen3.6-27B" + assert conf.model.exact_repo == "groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit" + assert not conf.model.allows_variant_selection + + def test_parses_base_model(self): + conf = parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": {"base": "Qwen/Qwen3.6-27B"}, + } + ) + + assert isinstance(conf.model, EndpointModelBase) + assert conf.model.api_model_name == "Qwen/Qwen3.6-27B" + assert conf.model.exact_repo is None + assert conf.model.allows_variant_selection + + def test_rejects_model_with_repo_and_base(self): + with pytest.raises(ConfigurationError): + parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": {"repo": "Qwen/Qwen3.6-27B", "base": "Qwen/Qwen3.6-27B"}, + } + ) + + def test_rejects_empty_model_repo(self): + with pytest.raises(ConfigurationError): + parse_apply_configuration( + { + "type": "endpoint", + "name": "qwen-endpoint", + "model": {"repo": ""}, + } + ) + def test_rejects_unknown_fields(self): with pytest.raises(ConfigurationError): parse_apply_configuration( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py index 253cf592ab..23a0401f24 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_endpoints.py @@ -12,6 +12,9 @@ from dstack._internal.core.models.configurations import ServiceConfiguration from dstack._internal.core.models.endpoints import ( EndpointConfiguration, + EndpointModelBase, + EndpointModelRepo, + EndpointModelSpec, EndpointPresetPolicy, EndpointStatus, ) @@ -93,6 +96,7 @@ async def _lock_endpoint_model(session: AsyncSession, endpoint_model: EndpointMo async def _create_endpoint_model( session: AsyncSession, status: EndpointStatus = EndpointStatus.SUBMITTED, + model: str | EndpointModelSpec = "Qwen/Qwen3-0.6B", user_ssh_public_key: str | None = None, user_global_role: GlobalRole = GlobalRole.ADMIN, project_role: ProjectRole | None = None, @@ -112,7 +116,7 @@ async def _create_endpoint_model( ) configuration = EndpointConfiguration( name="qwen-endpoint", - model="Qwen/Qwen3-0.6B", + model=EndpointModelRepo(repo=model) if isinstance(model, str) else model, env=Env.parse_obj({"HF_TOKEN": "secret"}), ) endpoint_model = EndpointModel( @@ -244,6 +248,8 @@ def _get_verified_agent_result( *, run_id: uuid.UUID | None = None, run_name: str | None = None, + base: str = "Qwen/Qwen3-0.6B", + model: str = "Qwen/Qwen3-0.6B", ) -> AgentProvisioningResult: return AgentProvisioningResult( run_id=run_id if run_id is not None else run.id, @@ -253,6 +259,8 @@ def _get_verified_agent_result( run_id=run.id, run_name=run.run_name, service_yaml=f"type: service\nname: {run.run_name}\n", + base=base, + model=model, ), ) @@ -263,8 +271,9 @@ def _get_agent_run_name(suffix: str = "1") -> str: def _make_preset_plan(run_name: str = "qwen-endpoint-serving") -> Mock: preset_plan = Mock() - preset_plan.preset.model = "Qwen/Qwen3-0.6B" + preset_plan.preset.base = "Qwen/Qwen3-0.6B" preset_plan.recipe.id = "vllm-t4" + preset_plan.recipe.model = "Qwen/Qwen3-0.6B" preset_plan.run_plan.run_spec = RunSpec( run_name=run_name, configuration=ServiceConfiguration.parse_obj( @@ -356,7 +365,7 @@ async def test_rejects_run_already_recorded_for_another_endpoint( endpoint_model = await _create_endpoint_model(session=session) other_configuration = EndpointConfiguration( name="other-qwen-endpoint", - model="Qwen/Qwen3-0.6B", + model=EndpointModelRepo(repo="Qwen/Qwen3-0.6B"), env=Env.parse_obj({"HF_TOKEN": "secret"}), ) other_endpoint_model = EndpointModel( @@ -1120,6 +1129,8 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): run_id=run.id, run_name=run.run_name, service_yaml=f"type: service\nname: {run.run_name}\n", + base="Qwen/Qwen3-0.6B", + model="Qwen/Qwen3-0.6B", ), ) @@ -1153,6 +1164,125 @@ async def provision_endpoint(endpoint_model, pipeline_hinter): assert submissions[0].run_id == endpoint_model.service_run_id preset_service.save_preset.assert_awaited_once() + async def test_agent_base_model_report_sets_endpoint_model_repo_and_preset_recipe_model( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROTOTYPING, + model=EndpointModelBase(base="Qwen/Qwen3-0.6B"), + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + selected_model = "groxaxo/Qwen3-0.6B-GPTQ-4Bit" + + async def provision_endpoint(endpoint_model, pipeline_hinter): + async with get_session_ctx() as provisioning_session: + res = await provisioning_session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_model.id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + provisioning_endpoint = res.unique().scalar_one() + run = await _create_backing_service_run( + session=provisioning_session, + endpoint_model=provisioning_endpoint, + link_endpoint=False, + run_name=_get_agent_run_name(), + ) + return AgentProvisioningResult( + run_id=run.id, + final_report=AgentFinalReport( + success=True, + run_id=run.id, + run_name=run.run_name, + service_yaml=f"type: service\nname: {run.run_name}\n", + base="Qwen/Qwen3-0.6B", + model=selected_model, + ), + ) + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + preset_service = Mock() + preset_service.save_preset = AsyncMock( + side_effect=lambda project_name, preset, comments: preset + ) + + with ( + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ), + patch( + "dstack._internal.server.background.pipeline_tasks.endpoints." + "get_endpoint_preset_service", + return_value=preset_service, + ), + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.RUNNING + assert endpoint_model.model_base == "Qwen/Qwen3-0.6B" + assert endpoint_model.model_repo == selected_model + saved_preset = preset_service.save_preset.await_args.args[1] + assert saved_preset.base == "Qwen/Qwen3-0.6B" + assert saved_preset.recipes[0].model == selected_model + + async def test_agent_exact_model_report_mismatch_fails_endpoint( + self, test_db, session: AsyncSession, worker: EndpointWorker + ): + endpoint_model = await _create_endpoint_model( + session=session, + status=EndpointStatus.PROTOTYPING, + ) + endpoint_model.provisioning_method = "agent" + await session.commit() + + async def provision_endpoint(endpoint_model, pipeline_hinter): + async with get_session_ctx() as provisioning_session: + res = await provisioning_session.execute( + select(EndpointModel) + .where(EndpointModel.id == endpoint_model.id) + .options(joinedload(EndpointModel.project)) + .options(joinedload(EndpointModel.user)) + ) + provisioning_endpoint = res.unique().scalar_one() + run = await _create_backing_service_run( + session=provisioning_session, + endpoint_model=provisioning_endpoint, + link_endpoint=False, + run_name=_get_agent_run_name(), + ) + return AgentProvisioningResult( + run_id=run.id, + final_report=AgentFinalReport( + success=True, + run_id=run.id, + run_name=run.run_name, + service_yaml=f"type: service\nname: {run.run_name}\n", + base="Qwen/Qwen3-0.6B", + model="groxaxo/Qwen3-0.6B-GPTQ-4Bit", + ), + ) + + agent_service = _FakeAgentService(side_effect=provision_endpoint) + + with patch( + "dstack._internal.server.background.pipeline_tasks.endpoints.get_agent_service", + return_value=agent_service, + ): + await worker.process(_endpoint_to_pipeline_item(endpoint_model)) + + await session.refresh(endpoint_model) + assert endpoint_model.status == EndpointStatus.FAILED + assert endpoint_model.service_run_id is None + assert endpoint_model.model_repo is None + assert endpoint_model.status_message == ( + "Server agent final report model does not match the requested model repo" + ) + async def test_agent_success_stops_non_final_submitted_runs( self, test_db, session: AsyncSession, worker: EndpointWorker ): @@ -1185,6 +1315,8 @@ async def test_agent_success_stops_non_final_submitted_runs( run_id=final_run.id, run_name=final_run.run_name, service_yaml=f"type: service\nname: {final_run.run_name}\n", + base="Qwen/Qwen3-0.6B", + model="Qwen/Qwen3-0.6B", ), ) ) @@ -1659,7 +1791,7 @@ async def test_saves_preset_when_agent_backing_service_becomes_running( preset_service.save_preset.assert_awaited_once() assert preset_service.save_preset.await_args.args[0] == "test_project" saved_preset = preset_service.save_preset.await_args.args[1] - assert saved_preset.model == "Qwen/Qwen3-0.6B" + assert saved_preset.base == "Qwen/Qwen3-0.6B" assert len(saved_preset.recipes) == 1 assert saved_preset.recipes[0].service.resources.gpu is not None assert saved_preset.recipes[0].validations[0].replicas[0].resources[0].gpu is not None diff --git a/src/tests/_internal/server/routers/test_endpoints.py b/src/tests/_internal/server/routers/test_endpoints.py index 20e12cd351..436a341a9f 100644 --- a/src/tests/_internal/server/routers/test_endpoints.py +++ b/src/tests/_internal/server/routers/test_endpoints.py @@ -84,7 +84,10 @@ async def test_returns_none_provisioning_plan( body = response.json() assert body["project_name"] == project.name assert body["user"] == user.name - assert body["configuration"]["model"] == "Qwen/Qwen3-0.6B" + assert body["configuration"]["model"] == { + "repo": "Qwen/Qwen3-0.6B", + "name": None, + } assert body["configuration_path"] == "endpoint.dstack.yml" assert body["current_resource"] is None assert body["action"] == "create" @@ -385,7 +388,7 @@ async def test_returns_preset_provisioning_plan( body = response.json() assert body["provisioning_plan"]["type"] == "preset" assert body["preset_policy"] == "reuse-or-create" - assert body["provisioning_plan"]["preset_model"] == "Qwen/Qwen3-0.6B" + assert body["provisioning_plan"]["preset_base"] == "Qwen/Qwen3-0.6B" assert body["provisioning_plan"]["recipe_id"] == "vllm-t4" assert body["provisioning_plan"]["service_name"] == "qwen-endpoint-serving" assert body["provisioning_plan"]["job_offers"][0]["replica_group"] == "0" @@ -432,7 +435,10 @@ async def test_creates_endpoint(self, test_db, session: AsyncSession, client: As assert body["name"] == "qwen-endpoint" assert body["project_name"] == project.name assert body["user"] == user.name - assert body["configuration"]["model"] == "Qwen/Qwen3-0.6B" + assert body["configuration"]["model"] == { + "repo": "Qwen/Qwen3-0.6B", + "name": None, + } assert body["configuration"]["env"] == {"HF_TOKEN": "secret"} assert body["created_at"] == "2023-01-02T03:04:00+00:00" assert body["last_processed_at"] == "2023-01-02T03:04:00+00:00" @@ -875,7 +881,7 @@ async def test_lists_endpoint_presets( assert response.status_code == 200, response.json() body = response.json() assert len(body) == 1 - assert body[0]["model"] == "Qwen/Qwen3-0.6B" + assert body[0]["base"] == "Qwen/Qwen3-0.6B" assert body[0]["recipes"][0]["id"] == "vllm-t4" assert body[0]["recipes"][0]["service"]["resources"]["gpu"] is not None assert body[0]["recipes"][0]["validations"][0]["replicas"][0]["resources"][0]["gpu"] @@ -939,7 +945,7 @@ async def test_gets_endpoint_preset( assert response.status_code == 200, response.json() body = response.json() - assert body["model"] == "Qwen/Qwen3-0.6B" + assert body["base"] == "Qwen/Qwen3-0.6B" assert body["recipes"][0]["service"]["type"] == "service" assert body["recipes"][0]["service"]["resources"]["gpu"] is not None assert preset_service.get_project_names == [project.name] @@ -960,12 +966,12 @@ async def list_presets(self, project_name): async def get_preset(self, project_name, model): self.get_project_names.append(project_name) for preset in self._presets: - if preset.model == model: + if preset.base == model: return preset return None async def delete_preset(self, project_name, model): - if model not in {preset.model for preset in self._presets}: + if model not in {preset.base for preset in self._presets}: raise FileNotFoundError(model) self.delete_project_names.append(project_name) self.deleted_models.append(model) @@ -1008,6 +1014,7 @@ def _endpoint_preset_plan() -> EndpointPresetPlan: ) recipe = EndpointPresetRecipe( id="vllm-t4", + model="Qwen/Qwen3-0.6B", service=service_configuration, validations=[ EndpointPresetValidation( @@ -1028,7 +1035,7 @@ def _endpoint_preset_plan() -> EndpointPresetPlan: ) ], ) - preset = EndpointPreset(model="Qwen/Qwen3-0.6B", recipes=[recipe]) + preset = EndpointPreset(base="Qwen/Qwen3-0.6B", recipes=[recipe]) run_plan = Mock() run_plan.get_effective_run_spec.return_value = RunSpec( run_name="qwen-endpoint-serving", diff --git a/src/tests/_internal/server/services/endpoints/test_agent_report.py b/src/tests/_internal/server/services/endpoints/test_agent_report.py index 99192254ba..a144ed387b 100644 --- a/src/tests/_internal/server/services/endpoints/test_agent_report.py +++ b/src/tests/_internal/server/services/endpoints/test_agent_report.py @@ -16,11 +16,15 @@ def test_accepts_success_report(self): "run_id": str(run_id), "run_name": "qwen-serving", "service_yaml": "type: service\nname: qwen-serving\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B", } ) assert report.run_id == run_id assert report.run_name == "qwen-serving" + assert report.base == "Qwen/Qwen3-0.6B" + assert report.model == "Qwen/Qwen3-0.6B" def test_rejects_success_without_run_id(self): with pytest.raises(ValidationError, match="run_id"): @@ -29,6 +33,34 @@ def test_rejects_success_without_run_id(self): "success": True, "run_name": "qwen-serving", "service_yaml": "type: service\nname: qwen-serving\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B", + } + ) + + def test_rejects_success_with_empty_base(self): + with pytest.raises(ValidationError, match="base"): + AgentFinalReport.parse_obj( + { + "success": True, + "run_id": str(uuid.uuid4()), + "run_name": "qwen-serving", + "service_yaml": "type: service\nname: qwen-serving\n", + "base": "", + "model": "Qwen/Qwen3-0.6B", + } + ) + + def test_rejects_success_with_empty_model(self): + with pytest.raises(ValidationError, match="model"): + AgentFinalReport.parse_obj( + { + "success": True, + "run_id": str(uuid.uuid4()), + "run_name": "qwen-serving", + "service_yaml": "type: service\nname: qwen-serving\n", + "base": "Qwen/Qwen3-0.6B", + "model": "", } ) diff --git a/src/tests/_internal/server/services/endpoints/test_claude_agent.py b/src/tests/_internal/server/services/endpoints/test_claude_agent.py index bb8f684728..90a8522c6f 100644 --- a/src/tests/_internal/server/services/endpoints/test_claude_agent.py +++ b/src/tests/_internal/server/services/endpoints/test_claude_agent.py @@ -44,6 +44,7 @@ async def _create_endpoint_model( session: AsyncSession, + model="Qwen/Qwen3-0.6B", max_price: float | None = None, spot_policy: SpotPolicy | None = None, backends: list[BackendType] | None = None, @@ -56,7 +57,7 @@ async def _create_endpoint_model( await create_fleet(session=session, project=project) configuration = EndpointConfiguration( name="qwen-endpoint", - model="Qwen/Qwen3-0.6B", + model=model, env=Env.parse_obj({"HF_TOKEN": "hf-secret"}), max_price=max_price, spot_policy=spot_policy, @@ -79,6 +80,28 @@ async def _create_endpoint_model( return endpoint_model +def _successful_agent_report( + *, + run_id: uuid.UUID | None = None, + run_name: str = "qwen-endpoint-1", + base: str = "Qwen/Qwen3-0.6B", + model: str = "Qwen/Qwen3-0.6B", + service_yaml: str | None = None, +) -> AgentFinalReport: + if run_id is None: + run_id = uuid.uuid4() + if service_yaml is None: + service_yaml = f"type: service\nname: {run_name}\n" + return AgentFinalReport( + success=True, + run_id=run_id, + run_name=run_name, + service_yaml=service_yaml, + base=base, + model=model, + ) + + def _configure_fake_claude(tmp_path, monkeypatch: pytest.MonkeyPatch) -> str: claude_path = tmp_path / "claude" claude_path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") @@ -271,14 +294,7 @@ async def test_new_endpoint_lifecycle_starts_new_session_without_previous_contex async def runner(workspace, request): captured["workspace"] = workspace captured["request"] = request - return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=uuid.uuid4(), - run_name="qwen-endpoint-1", - service_yaml="type: service\nname: qwen-endpoint-1\n", - ) - ) + return _AgentRunnerResult(report=_successful_agent_report()) service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) @@ -465,14 +481,7 @@ async def test_invokes_agent_with_isolated_dstack_cli_context( async def runner(workspace, request): captured["workspace"] = workspace captured["request"] = request - return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=run_id, - run_name="qwen-endpoint-1", - service_yaml="type: service\nname: qwen-endpoint-1\n", - ) - ) + return _AgentRunnerResult(report=_successful_agent_report(run_id=run_id)) endpoint_model = await _create_endpoint_model(session) service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) @@ -567,6 +576,83 @@ async def runner(workspace, request): assert final_report["success"] is True assert final_report["run_name"] == "qwen-endpoint-1" + @pytest.mark.asyncio + async def test_prompt_includes_base_model_endpoint_context( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + captured = {} + + async def runner(workspace, request): + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=False, + failure_summary="not needed", + ) + ) + + endpoint_model = await _create_endpoint_model( + session, + model={"base": "Qwen/Qwen3.6-27B"}, + ) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.final_report is not None + prompt = captured["request"]["prompt"] + assert "Endpoint context:" in prompt + assert "- endpoint_name: qwen-endpoint" in prompt + assert "- service_model_name: Qwen/Qwen3.6-27B" in prompt + assert "- base_model: Qwen/Qwen3.6-27B" in prompt + assert "- model_repo:" not in prompt + assert "- final_report.json.model:" not in prompt + + @pytest.mark.asyncio + async def test_prompt_includes_exact_model_endpoint_context( + self, + session: AsyncSession, + tmp_path, + monkeypatch, + ): + monkeypatch.setattr(settings, "AGENT_ANTHROPIC_API_KEY", "agent-secret") + _configure_fake_claude(tmp_path, monkeypatch) + captured = {} + + async def runner(workspace, request): + captured["request"] = request + return _AgentRunnerResult( + report=AgentFinalReport( + success=False, + failure_summary="not needed", + ) + ) + + endpoint_model = await _create_endpoint_model( + session, + model={ + "repo": "groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit", + "name": "Qwen/Qwen3.6-27B", + }, + ) + service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) + + result = await service.provision_endpoint(endpoint_model, pipeline_hinter=None) + + assert result.final_report is not None + prompt = captured["request"]["prompt"] + assert "Endpoint context:" in prompt + assert "- endpoint_name: qwen-endpoint" in prompt + assert "- service_model_name: Qwen/Qwen3.6-27B" in prompt + assert "- model_repo: groxaxo/Qwen3.6-27B-GPTQ-Pro-4Bit" in prompt + assert "- base_model:" not in prompt + assert "- final_report.json.model:" not in prompt + @pytest.mark.asyncio async def test_existing_claude_auth_uses_real_home_for_claude_and_short_home_for_dstack( self, @@ -585,14 +671,7 @@ async def test_existing_claude_auth_uses_real_home_for_claude_and_short_home_for async def runner(workspace, request): captured["workspace"] = workspace captured["request"] = request - return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=run_id, - run_name="qwen-endpoint-1", - service_yaml="type: service\nname: qwen-endpoint-1\n", - ) - ) + return _AgentRunnerResult(report=_successful_agent_report(run_id=run_id)) endpoint_model = await _create_endpoint_model(session) service = ClaudeAgentService(runner=runner, workspace_base_dir=tmp_path) @@ -636,14 +715,7 @@ async def test_includes_endpoint_profile_constraints_in_prompt( async def runner(workspace, request): captured["request"] = request - return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=uuid.uuid4(), - run_name="qwen-endpoint-1", - service_yaml="type: service\nname: qwen-endpoint-1\n", - ) - ) + return _AgentRunnerResult(report=_successful_agent_report()) endpoint_model = await _create_endpoint_model( session, @@ -683,6 +755,8 @@ async def test_reuses_existing_final_report_without_invoking_runner( "run_id": str(run_id), "run_name": "qwen-endpoint-1", "service_yaml": "type: service\nname: qwen-endpoint-1\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B", } ), encoding="utf-8", @@ -726,6 +800,8 @@ async def test_waits_for_claude_result_after_final_report_while_process_runs( "run_id": str(uuid.uuid4()), "run_name": "qwen-endpoint-1", "service_yaml": "type: service\nname: qwen-endpoint-1\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B", } ), encoding="utf-8", @@ -784,12 +860,7 @@ async def runner(workspace, request): {"type": "FakeMessage", "text": "hf-secret agent-secret"}, ) return _AgentRunnerResult( - report=AgentFinalReport( - success=True, - run_id=run_id, - run_name="qwen-endpoint-1", - service_yaml="token: hf-secret\n", - ) + report=_successful_agent_report(run_id=run_id, service_yaml="token: hf-secret\n") ) endpoint_model = await _create_endpoint_model(session) @@ -1035,6 +1106,8 @@ async def test_reads_claude_stream_incrementally(self, tmp_path): "run_id": str(uuid.uuid4()), "run_name": "qwen-endpoint-1", "service_yaml": "env: secret\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B", } reader.feed_data( json.dumps({"type": "assistant", "message": {"content": "working"}}).encode() + b"\n" @@ -1250,7 +1323,9 @@ async def flush(self): "success": true, "run_id": "6e578748-d597-4fde-a3a4-203587cad5a2", "run_name": "qwen-endpoint-1", - "service_yaml": "type: service\\nname: qwen-endpoint-1\\n" + "service_yaml": "type: service\\nname: qwen-endpoint-1\\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B" } JSON printf '%s\\n' '{"type":"result","is_error":false,"result":"done"}' @@ -1312,7 +1387,9 @@ async def test_subprocess_uses_final_report_artifact_when_stream_result_is_missi "success": true, "run_id": "6e578748-d597-4fde-a3a4-203587cad5a2", "run_name": "qwen-endpoint-1", - "service_yaml": "type: service\\nname: qwen-endpoint-1\\n" + "service_yaml": "type: service\\nname: qwen-endpoint-1\\n", + "base": "Qwen/Qwen3-0.6B", + "model": "Qwen/Qwen3-0.6B" } JSON printf '%s\\n' '{"type":"result","is_error":false,"result":"done"}' diff --git a/src/tests/_internal/server/services/test_endpoint_presets.py b/src/tests/_internal/server/services/test_endpoint_presets.py index 2b9a104280..93f9e6bc9f 100644 --- a/src/tests/_internal/server/services/test_endpoint_presets.py +++ b/src/tests/_internal/server/services/test_endpoint_presets.py @@ -83,7 +83,7 @@ async def test_lists_valid_endpoint_presets(self, tmp_path): presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) assert len(presets) == 1 - assert presets[0].model == "Qwen/Qwen3-4B" + assert presets[0].base == "Qwen/Qwen3-4B" assert [recipe.id for recipe in presets[0].recipes] == ["vllm-t4"] recipe = presets[0].recipes[0] assert recipe.service.resources.gpu is not None @@ -223,7 +223,7 @@ async def test_rejects_service_gpu_vendor_mismatch(self, tmp_path, caplog): assert "preset service GPU vendor does not match validation" in caplog.text @pytest.mark.asyncio - async def test_saves_preset_with_comments_and_merges_same_model(self, tmp_path): + async def test_saves_preset_with_comments_and_merges_same_base(self, tmp_path): preset = _qwen_preset(recipe_id="vllm-t4") extra_preset = _qwen_preset(recipe_id="vllm-l4", resources={"gpu": "24GB"}) service = LocalDirEndpointPresetService(tmp_path) @@ -236,7 +236,7 @@ async def test_saves_preset_with_comments_and_merges_same_model(self, tmp_path): "run: 00000000-0000-0000-0000-000000000001", ], ) - assert saved.model == "Qwen/Qwen3-4B" + assert saved.base == "Qwen/Qwen3-4B" presets_dir = _project_presets_dir(tmp_path) preset_files = list(presets_dir.glob("*.dstack.yml")) assert len(preset_files) == 1 @@ -248,6 +248,7 @@ async def test_saves_preset_with_comments_and_merges_same_model(self, tmp_path): assert "preset-service-name" not in text assert "creation_policy" not in text assert "validations:" in text + assert saved.recipes[0].model == "Qwen/Qwen3-4B" assert "HF_HOME=/root/.cache/huggingface" in text assert "HF_TOKEN" in text saved_again = await service.save_preset(PROJECT_NAME, extra_preset) @@ -256,7 +257,7 @@ async def test_saves_preset_with_comments_and_merges_same_model(self, tmp_path): assert len(list(presets_dir.glob("*.dstack.yml"))) == 1 @pytest.mark.asyncio - async def test_merges_duplicate_model_files_for_list_get_save_and_delete(self, tmp_path): + async def test_merges_duplicate_base_files_for_list_get_save_and_delete(self, tmp_path): service = LocalDirEndpointPresetService(tmp_path) presets_dir = _project_presets_dir(tmp_path) presets_dir.mkdir(parents=True) @@ -293,22 +294,83 @@ async def test_saved_preset_round_trips(self, tmp_path): presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) - assert saved.model == "Qwen/Qwen3-4B" + assert saved.base == "Qwen/Qwen3-4B" assert len(presets) == 1 - assert presets[0].model == "Qwen/Qwen3-4B" + assert presets[0].base == "Qwen/Qwen3-4B" recipe = presets[0].recipes[0] + assert recipe.model == "Qwen/Qwen3-4B" assert recipe.service.model is not None assert recipe.service.model.name == "Qwen/Qwen3-4B" assert set(recipe.service.env.keys()) == {"HF_HOME", "HF_TOKEN"} assert recipe.service.env["HF_HOME"] == "/root/.cache/huggingface" assert recipe.service.resources.gpu is not None + @pytest.mark.asyncio + async def test_loads_recipe_model_that_differs_from_preset_base(self, tmp_path): + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + (presets_dir / "qwen.yml").write_text( + """\ +type: endpoint-preset +model: Qwen/Qwen3-4B +recipes: + - id: vllm-t4 + model: groxaxo/Qwen3-4B-GPTQ-4Bit + service: + commands: + - vllm serve groxaxo/Qwen3-4B-GPTQ-4Bit --served-model-name Qwen/Qwen3-4B + port: 8000 + model: Qwen/Qwen3-4B + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: + name: T4 + memory: 16GB + count: 1 +""" + ) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) + + recipe = presets[0].recipes[0] + assert recipe.model == "groxaxo/Qwen3-4B-GPTQ-4Bit" + assert recipe.service.model is not None + assert recipe.service.model.name == "Qwen/Qwen3-4B" + + @pytest.mark.asyncio + async def test_legacy_recipe_without_model_uses_preset_base(self, tmp_path): + _write_qwen_preset(tmp_path) + + presets = await LocalDirEndpointPresetService(tmp_path).list_presets(PROJECT_NAME) + + assert presets[0].recipes[0].model == "Qwen/Qwen3-4B" + + @pytest.mark.asyncio + async def test_rejects_same_recipe_id_with_different_recipe_model(self, tmp_path): + service = LocalDirEndpointPresetService(tmp_path) + await service.save_preset( + PROJECT_NAME, + _qwen_preset(recipe_id="vllm-t4", recipe_model="Qwen/Qwen3-4B-GPTQ-A"), + ) + + with pytest.raises(ValueError, match="endpoint preset recipe id conflict"): + await service.save_preset( + PROJECT_NAME, + _qwen_preset(recipe_id="vllm-t4", recipe_model="Qwen/Qwen3-4B-GPTQ-B"), + ) + @pytest.mark.asyncio async def test_deletes_preset_by_model(self, tmp_path): service = LocalDirEndpointPresetService(tmp_path) saved = await service.save_preset(PROJECT_NAME, _qwen_preset()) - await service.delete_preset(PROJECT_NAME, saved.model) + await service.delete_preset(PROJECT_NAME, saved.base) assert await service.list_presets(PROJECT_NAME) == [] assert list(_project_presets_dir(tmp_path).glob("*.dstack.yml")) == [] @@ -340,7 +402,7 @@ async def test_ignores_non_yaml_files(self, tmp_path): "preset must be an endpoint preset", ), ( - "missing-model.yml", + "missing-base.yml", """\ type: endpoint-preset recipes: @@ -359,7 +421,31 @@ async def test_ignores_non_yaml_files(self, tmp_path): disk: 100GB gpu: 0 """, - "preset must specify a model", + "preset must specify a base", + ), + ( + "base-model-mismatch.yml", + """\ +type: endpoint-preset +base: Qwen/Qwen3-4B +model: Qwen/Qwen3-8B +recipes: + - id: vllm + service: + commands: + - python -m http.server 8000 + port: 8000 + resources: + gpu: 16GB + validations: + - replicas: + - resources: + - cpu: 4 + memory: 16GB + disk: 100GB + gpu: 0 +""", + "preset base must match legacy model", ), ( "missing-recipes.yml", @@ -597,10 +683,17 @@ async def test_builds_implicit_replica_group_from_running_service( await session.refresh(run, attribute_names=["jobs"]) preset = build_endpoint_preset_from_run(run) + variant_preset = build_endpoint_preset_from_run( + run, + recipe_model="groxaxo/Qwen3-4B-GPTQ-4Bit", + ) saved = await LocalDirEndpointPresetService(tmp_path).save_preset(PROJECT_NAME, preset) - assert saved.model == "Qwen/Qwen3-4B" + assert saved.base == "Qwen/Qwen3-4B" + assert variant_preset.base == "Qwen/Qwen3-4B" + assert variant_preset.recipes[0].model == "groxaxo/Qwen3-4B-GPTQ-4Bit" recipe = saved.recipes[0] + assert recipe.model == "Qwen/Qwen3-4B" assert recipe.id == make_endpoint_preset_recipe_id(recipe.service) assert recipe.service.resources.gpu is not None assert recipe.service.resources.gpu.vendor is not None @@ -835,12 +928,51 @@ async def test_returns_first_preset_with_available_offers( ) assert match is not None - assert match.preset.model == "Qwen/Qwen3-4B" + assert match.preset.base == "Qwen/Qwen3-4B" assert match.recipe.id == "vllm-t4" get_plan_mock.assert_awaited_once() assert get_plan_mock.await_args is not None assert get_plan_mock.await_args.kwargs["run_spec"].run_name == "qwen-endpoint-serving" + @pytest.mark.asyncio + async def test_exact_repo_request_skips_other_recipe_models( + self, session: AsyncSession, tmp_path + ): + presets_dir = _project_presets_dir(tmp_path) + presets_dir.mkdir(parents=True) + _write_qwen_preset_file( + presets_dir / "qwen-a.yml", + recipe_id="variant", + recipe_model="groxaxo/Qwen3-4B-GPTQ-4Bit", + ) + _write_qwen_preset_file(presets_dir / "qwen-b.yml", recipe_id="base") + user = await create_user( + session=session, + global_role=GlobalRole.USER, + ssh_public_key="ssh-rsa test", + ) + project = await create_project(session=session, owner=user) + + with patch( + "dstack._internal.server.services.endpoints.planning.runs_services.get_plan", + new=AsyncMock(return_value=_run_plan_with_offer(available=True)), + ) as get_plan_mock: + match = await find_matching_preset_plan( + session=session, + project=project, + user=user, + endpoint_name="qwen-endpoint", + endpoint_configuration=EndpointConfiguration( + name="qwen-endpoint", + model={"repo": "Qwen/Qwen3-4B"}, + ), + preset_service=LocalDirEndpointPresetService(tmp_path), + ) + + assert match is not None + assert match.recipe.id == "base" + get_plan_mock.assert_awaited_once() + @pytest.mark.asyncio async def test_tracks_first_unprovisionable_preset_when_offers_are_unavailable( self, session: AsyncSession, tmp_path @@ -871,7 +1003,7 @@ async def test_tracks_first_unprovisionable_preset_when_offers_are_unavailable( assert result.provisionable is None assert result.unprovisionable is not None - assert result.unprovisionable.preset.model == "Qwen/Qwen3-4B" + assert result.unprovisionable.preset.base == "Qwen/Qwen3-4B" assert result.unprovisionable.recipe.id == "vllm-t4" @pytest.mark.asyncio @@ -1082,13 +1214,16 @@ def _write_qwen_preset_file( path, recipe_id: str = "vllm-t4", gpu: str = "16GB", + recipe_model: str | None = None, ): + recipe_model_line = f" model: {recipe_model}\n" if recipe_model is not None else "" path.write_text( """\ type: endpoint-preset model: Qwen/Qwen3-4B recipes: - id: {recipe_id} +{recipe_model_line}\ service: commands: - vllm serve Qwen/Qwen3-4B --host 0.0.0.0 --port 8000 @@ -1105,7 +1240,7 @@ def _write_qwen_preset_file( name: T4 memory: 16GB count: 1 -""".format(recipe_id=recipe_id, gpu=gpu) +""".format(recipe_id=recipe_id, gpu=gpu, recipe_model_line=recipe_model_line) ) @@ -1149,6 +1284,7 @@ def _instance_offer( def _qwen_preset( recipe_id: str = "vllm-t4", resources: dict | None = None, + recipe_model: str = "Qwen/Qwen3-4B", ) -> EndpointPreset: service = ServiceConfiguration.parse_obj( { @@ -1168,10 +1304,11 @@ def _qwen_preset( } ) return EndpointPreset( - model="Qwen/Qwen3-4B", + base="Qwen/Qwen3-4B", recipes=[ EndpointPresetRecipe( id=recipe_id, + model=recipe_model, service=service, validations=[ EndpointPresetValidation(