Improve dbt ls error logging

astronomer · Jul 14, 2023 · 2d268e7 · 2d268e7
1 parent e72946e
commit 2d268e7
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 9 deletions.
diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py
@@ -76,9 +76,9 @@ def load(self, method=LoadMode.AUTOMATIC, execution_mode="local"):
             elif execution_mode in ("local", "virtualenv"):
                 try:
                     self.load_via_dbt_ls()
+                    return
                 except FileNotFoundError:
                     self.load_via_custom_parser()
-                finally:
                     return
             else:
                 self.load_via_custom_parser()
@@ -96,19 +96,22 @@ def load_via_dbt_ls(self):
             command.extend(["--exclude", *self.exclude])
         if self.select:
             command.extend(["--select", *self.select])
-
+        logger.info(f"Running command: {command}")
         try:
-            process = Popen(command, stdout=PIPE, stderr=PIPE, cwd=self.project.dir)
+            process = Popen(command, stdout=PIPE, stderr=PIPE, cwd=self.project.dir, universal_newlines=True)
         except FileNotFoundError as exception:
-            raise CosmosLoadDbtException(f"Unable to run the command {command} due to the error:\n{exception}")
+            raise CosmosLoadDbtException(f"Unable to run the command due to the error:\n{exception}")
 
         stdout, stderr = process.communicate()
-        # TODO: cover this with test:
-        if stderr:
-            raise CosmosLoadDbtException(f"Unable to run the command {command} due to the error:\n{stderr}")
+
+        logger.debug(f"Output: {stdout}")
+
+        if stderr or "Runtime Error" in stdout:
+            details = stderr or stdout
+            raise CosmosLoadDbtException(f"Unable to run the command due to the error:\n{details}")
 
         nodes = {}
-        for line in stdout.decode().split("\n"):
+        for line in stdout.split("\n"):
             try:
                 node_dict = json.loads(line.strip())
             except json.decoder.JSONDecodeError:

diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py
@@ -40,5 +40,7 @@ def update_profile_path(self):
             self.profile_path = self.pipeline_dir / "profiles.yml"
         if not self.profile_path.exists():
             with open(self.profile_path, "w") as fp:
-                profile_content = {self.name: {"target": "dev"}, "output": {"dev": {}}}
+                profile_content = {
+                    self.name: {"target": "dev", "outputs": {"dev": {"type": "sqlite", "database": "/tmp/dummy.db"}}}
+                }
                 yaml.dump(profile_content, fp)