Path de duckdb configurable, correction des tests #72

.template.env

@@ -1,4 +1,5 @@
 DATA_FILE_PARQUET_PATH=https://www.data.gouv.fr/fr/datasets/r/11cea8e8-df3e-4ed1-932b-781e2635e432
+DUCKDB_PATH=./decp.duckdb
 PORT=8050
 DEVELOPMENT=True
 SOURCE_STATS_CSV_PATH="https://www.data.gouv.fr/api/1/datasets/r/8ded94de-3b80-4840-a5bb-7faad1c9c234"

pyproject.toml

@@ -40,6 +40,7 @@ testpaths = ["tests"]
 env = [
   "DATA_FILE_PARQUET_PATH=tests/test.parquet",
   "DEVELOPMENT=true",
+  "REBUILD_DUCKDB=true",
   "DATA_SCHEMA_PATH=/home/colin/git/decp-processing/dist/schema.json",
 ]
 addopts = "-p no:warnings"

src/db.py

@@ -110,7 +110,7 @@ def build_database(db_path: Path, parquet_path: Path) -> None:
 
 
 def _ensure_database() -> Path:
-    db_path = Path("./decp.duckdb")
+    db_path = Path(os.getenv("DUCKDB_PATH", "./decp.duckdb"))
     parquet_path = Path(os.getenv("DATA_FILE_PARQUET_PATH"))
     lock_path = db_path.with_suffix(".duckdb.lock")
 

tests/conftest.py

@@ -6,10 +6,7 @@ import polars as pl
 import pytest
 from selenium.webdriver.chrome.options import Options
 
-
+_TEST_DATA = [
-@pytest.fixture(scope="session", autouse=True)
-def test_data():
-    data = [
     {
         "uid": "1",
         "id": "1",
@@ -41,20 +38,34 @@ def test_data():
         "acheteur_categorie": "Collectivité",
         "titulaire_categorie": "PME",
     }
-    ]
+]
-    parquet_path = Path(os.path.abspath("tests/test.parquet"))
+_PARQUET_PATH = Path(os.path.abspath("tests/test.parquet"))
-    db_path = parquet_path.parent / "decp.duckdb"
+_DB_PATH = Path(os.path.abspath("decp.duckdb"))
-    print(f"Writing test data to: {parquet_path}")
 
-    pl.DataFrame(data).write_parquet(parquet_path)
 
-    # Remove any stale DuckDB from a previous run so src.db rebuilds from
+def _cleanup_db_artifacts() -> None:
-    # the freshly-written parquet at import time.
+    for artifact in (
-    for artifact in (db_path, db_path.with_suffix(".duckdb.tmp")):
+        _DB_PATH,
+        _DB_PATH.with_suffix(".duckdb.tmp"),
+        _DB_PATH.with_suffix(".duckdb.lock"),
+    ):
         if artifact.exists():
             artifact.unlink()
 
-    yield str(parquet_path)
+
+# Runs at conftest import, before test modules import src.db (which builds the
+# DuckDB at import time). Guarantees the test parquet exists and the stale DB
+# from a previous `python run.py` is wiped so src.db rebuilds from test data.
+pl.DataFrame(_TEST_DATA).write_parquet(_PARQUET_PATH)
+_cleanup_db_artifacts()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def test_data():
+    yield str(_PARQUET_PATH)
+    # Teardown: remove the test DuckDB so the next `python run.py` rebuilds
+    # from decp_prod.parquet.
+    _cleanup_db_artifacts()
 
 
 def pytest_setup_options():

tests/test_db.py

@@ -141,6 +141,7 @@ def built_db(tmp_path, monkeypatch):
     )
     data.write_parquet(parquet_path)
     monkeypatch.setenv("DATA_FILE_PARQUET_PATH", str(parquet_path))
+    monkeypatch.setenv("DUCKDB_PATH", str(db_path))
 
     from src.db import build_database