From 8babfb50863de042e01b442784c5e7641bfde837 Mon Sep 17 00:00:00 2001 From: nk Date: Fri, 12 Jun 2026 19:26:04 +0900 Subject: [PATCH 1/4] fix(dlt): surface underlying error and hint when attaching to pipeline fails When `sqlmesh init -t dlt --dlt-path ` could not attach to the pipeline, the underlying dlt error was swallowed and replaced with an uninformative message. `--dlt-path` points to dlt's pipelines working directory (by default ~/.dlt/pipelines), not the directory containing the pipeline scripts, which is a common source of confusion. Now the error includes the original dlt exception, the directory that was searched, and a hint to omit `--dlt-path` when the pipeline exists in the default working directory. Also clarifies the `--dlt-path` help text and docstring accordingly. Fixes #5660. Co-Authored-By: Claude Fable 5 Signed-off-by: nk --- docs/reference/cli.md | 3 ++- sqlmesh/cli/main.py | 4 ++-- sqlmesh/integrations/dlt.py | 20 +++++++++++++++++--- tests/cli/test_cli.py | 8 +++++++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index a6d4fa9514..b65f8256ac 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -279,7 +279,8 @@ Options: empty. --dlt-pipeline TEXT DLT pipeline for which to generate a SQLMesh project. Use alongside template: dlt - --dlt-path TEXT The directory where the DLT pipeline resides. Use + --dlt-path TEXT The DLT pipelines working directory, where DLT stores + pipeline state (by default ~/.dlt/pipelines). Use alongside template: dlt --help Show this message and exit. ``` diff --git a/sqlmesh/cli/main.py b/sqlmesh/cli/main.py index c19d2ca629..b3c7a7027b 100644 --- a/sqlmesh/cli/main.py +++ b/sqlmesh/cli/main.py @@ -169,7 +169,7 @@ def cli( @click.option( "--dlt-path", type=str, - help="The directory where the DLT pipeline resides. Use alongside template: dlt", + help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines). Use alongside template: dlt", ) @click.pass_context @error_handler @@ -1155,7 +1155,7 @@ def table_name( @click.option( "--dlt-path", type=str, - help="The directory where the DLT pipeline resides.", + help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines).", ) @click.pass_context @error_handler diff --git a/sqlmesh/integrations/dlt.py b/sqlmesh/integrations/dlt.py index 2d601a0e22..a2202bea02 100644 --- a/sqlmesh/integrations/dlt.py +++ b/sqlmesh/integrations/dlt.py @@ -22,7 +22,8 @@ def generate_dlt_models_and_settings( pipeline_name: The name of the DLT pipeline to attach to. dialect: The SQL dialect to use for generating SQLMesh models. tables: A list of table names to include. - dlt_path: The path to the directory containing the DLT pipelines. + dlt_path: The path to the DLT pipelines working directory, where DLT stores + pipeline state (by default ~/.dlt/pipelines). Returns: A tuple containing a set of the SQLMesh model definitions, the connection config and the start date. @@ -34,8 +35,21 @@ def generate_dlt_models_and_settings( try: pipeline = dlt.attach(pipeline_name=pipeline_name, pipelines_dir=dlt_path or "") - except CannotRestorePipelineException: - raise click.ClickException(f"Could not attach to pipeline {pipeline_name}") + except CannotRestorePipelineException as e: + from pathlib import Path + from dlt.common.pipeline import get_dlt_pipelines_dir + + searched_dir = dlt_path or get_dlt_pipelines_dir() + msg = f"Could not attach to pipeline {pipeline_name}.\nSearched in: {searched_dir}\n{e}" + if dlt_path and (Path(get_dlt_pipelines_dir()) / pipeline_name).exists(): + msg += ( + f"\nHint: A pipeline named '{pipeline_name}' exists in the default pipelines " + f"working directory '{get_dlt_pipelines_dir()}'. Note that --dlt-path must " + "point to the directory where DLT stores pipeline working state (by default " + "~/.dlt/pipelines), not the directory containing your pipeline scripts. " + "Try omitting --dlt-path." + ) + raise click.ClickException(msg) schema = pipeline.default_schema dataset = pipeline.dataset_name diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index ba71e35843..938f90cc74 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -995,7 +995,7 @@ def test_dlt_pipeline(runner, tmp_path): exec(file.read()) # This should fail since it won't be able to locate the pipeline in this path - with pytest.raises(ClickException, match=r".*Could not attach to pipeline*"): + with pytest.raises(ClickException, match=r".*Could not attach to pipeline*") as excinfo: init_example_project( tmp_path, "duckdb", @@ -1004,6 +1004,12 @@ def test_dlt_pipeline(runner, tmp_path): dlt_path="./dlt2/pipelines", ) + # The error should surface where the pipeline was searched for and, since the + # pipeline exists in the default working directory, a hint about --dlt-path + error_message = str(excinfo.value) + assert "Searched in: ./dlt2/pipelines" in error_message + assert "Try omitting --dlt-path" in error_message + # By setting the pipelines path where the pipeline directory is located, it should work dlt_path = get_dlt_pipelines_dir() init_example_project( From a439ef664dc66efc30dafb89ee3e180096e91c56 Mon Sep 17 00:00:00 2001 From: nk Date: Fri, 19 Jun 2026 21:04:13 +0900 Subject: [PATCH 2/4] docs(dlt): clarify dlt path in docs and magics Signed-off-by: nk --- docs/integrations/dlt.md | 10 +++++----- sqlmesh/magics.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/integrations/dlt.md b/docs/integrations/dlt.md index d8d38cb864..ffa5e87754 100644 --- a/docs/integrations/dlt.md +++ b/docs/integrations/dlt.md @@ -28,12 +28,12 @@ This will create the configuration file and directories, which are found in all SQLMesh will also automatically generate models to ingest data from the pipeline incrementally. Incremental loading is ideal for large datasets where recomputing entire tables is resource-intensive. In this case utilizing the [`INCREMENTAL_BY_TIME_RANGE` model kind](../concepts/models/model_kinds.md#incremental_by_time_range). However, these model definitions can be customized to meet your specific project needs. -#### Specify the path to the pipelines directory +#### Specify the path to the pipelines working directory -The default location for dlt pipelines is `~/.dlt/pipelines/`. If your pipelines are in a [different directory](https://dlthub.com/docs/general-usage/pipeline#separate-working-environments-with-pipelines_dir), use the `--dlt-path` argument to specify the path explicitly: +The default location for dlt pipeline working state is `~/.dlt/pipelines/`. If dlt stores your pipeline state in a [different pipelines working directory](https://dlthub.com/docs/general-usage/pipeline#separate-working-environments-with-pipelines_dir), use the `--dlt-path` argument to specify that directory explicitly. This should be the directory where dlt stores pipeline state, not the directory containing your pipeline scripts: ```bash -sqlmesh init -t dlt --dlt-pipeline --dlt-path dialect +sqlmesh init -t dlt --dlt-pipeline --dlt-path dialect ``` ### Generating models on demand @@ -58,10 +58,10 @@ sqlmesh dlt_refresh --force sqlmesh dlt_refresh --table ``` -- **Provide the explicit path to the pipelines directory** (using `--dlt-path`): +- **Provide the explicit path to the pipelines working directory** (using `--dlt-path`): ```bash -sqlmesh dlt_refresh --dlt-path +sqlmesh dlt_refresh --dlt-path ``` #### Configuration diff --git a/sqlmesh/magics.py b/sqlmesh/magics.py index 57dd150af2..3a59fc4f7b 100644 --- a/sqlmesh/magics.py +++ b/sqlmesh/magics.py @@ -232,7 +232,7 @@ def context(self, line: str) -> None: @argument( "--dlt-path", type=str, - help="The directory where the DLT pipeline resides. Use alongside template: dlt", + help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines). Use alongside template: dlt", ) @line_magic def init(self, line: str) -> None: @@ -886,7 +886,7 @@ def table_name(self, context: Context, line: str) -> None: @argument( "--dlt-path", type=str, - help="The directory where the DLT pipeline resides.", + help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines).", ) @line_magic @pass_sqlmesh_context From 5ba605e55742ec95b1d47567900113bea92acadb Mon Sep 17 00:00:00 2001 From: nk Date: Sat, 20 Jun 2026 00:07:32 +0900 Subject: [PATCH 3/4] ci: pin pyopenssl for dbt 1.6 tests Signed-off-by: nk --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3892731352..6bf02309b3 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ install-dev-dbt-%: $(MAKE) install-dev; \ if [ "$$version" = "1.6.0" ]; then \ echo "Applying overrides for dbt 1.6.0"; \ - $(PIP) install 'pydantic>=2.0.0' 'google-cloud-bigquery==3.30.0' 'databricks-sdk==0.28.0' --reinstall; \ + $(PIP) install 'pydantic>=2.0.0' 'google-cloud-bigquery==3.30.0' 'databricks-sdk==0.28.0' 'pyOpenSSL>=25.3.0' --reinstall; \ fi; \ if [ "$$version" = "1.7.0" ]; then \ echo "Applying overrides for dbt 1.7.0"; \ From 2b3da1a5e95d3768697588058ca27569775e9db3 Mon Sep 17 00:00:00 2001 From: nk Date: Sat, 20 Jun 2026 00:11:02 +0900 Subject: [PATCH 4/4] ci: pin pyopenssl for dbt 1.7 tests Signed-off-by: nk --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6bf02309b3..26f7375c73 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ install-dev-dbt-%: fi; \ if [ "$$version" = "1.7.0" ]; then \ echo "Applying overrides for dbt 1.7.0"; \ - $(PIP) install 'databricks-sdk==0.28.0' --reinstall; \ + $(PIP) install 'databricks-sdk==0.28.0' 'pyOpenSSL>=25.3.0' --reinstall; \ fi; \ if [ "$$version" = "1.5.0" ]; then \ echo "Applying overrides for dbt 1.5.0"; \