Path de duckdb configurable, correction des tests #72

.template.env

@@ -1,4 +1,5 @@
 DATA_FILE_PARQUET_PATH=https://www.data.gouv.fr/fr/datasets/r/11cea8e8-df3e-4ed1-932b-781e2635e432
+DUCKDB_PATH=./decp.duckdb
 PORT=8050
 DEVELOPMENT=True
 SOURCE_STATS_CSV_PATH="https://www.data.gouv.fr/api/1/datasets/r/8ded94de-3b80-4840-a5bb-7faad1c9c234"

pyproject.toml

@@ -40,6 +40,7 @@ testpaths = ["tests"]
 env = [
   "DATA_FILE_PARQUET_PATH=tests/test.parquet",
   "DEVELOPMENT=true",
+  "REBUILD_DUCKDB=true",
   "DATA_SCHEMA_PATH=/home/colin/git/decp-processing/dist/schema.json",
 ]
 addopts = "-p no:warnings"

src/db.py

@@ -110,7 +110,7 @@ def build_database(db_path: Path, parquet_path: Path) -> None:
 
 
 def _ensure_database() -> Path:
-    db_path = Path("./decp.duckdb")
+    db_path = Path(os.getenv("DUCKDB_PATH", "./decp.duckdb"))
     parquet_path = Path(os.getenv("DATA_FILE_PARQUET_PATH"))
     lock_path = db_path.with_suffix(".duckdb.lock")
 

tests/conftest.py

@@ -6,55 +6,66 @@ import polars as pl
 import pytest
 from selenium.webdriver.chrome.options import Options
 
+_TEST_DATA = [
+    {
+        "uid": "1",
+        "id": "1",
+        "acheteur_nom": "ACHETEUR 1",
+        "acheteur_id": "123",
+        "titulaire_nom": "TITULAIRE 1",
+        "titulaire_id": "345",
+        "montant": 10,
+        "dateNotification": datetime.date(2025, 1, 1),
+        "codeCPV": "71600000",
+        "donneesActuelles": True,
+        "acheteur_departement_code": "75",
+        "acheteur_departement_nom": "Paris",
+        "acheteur_commune_nom": "Paris",
+        "titulaire_departement_code": "35",
+        "titulaire_departement_nom": "Ille-et-Vilaine",
+        "titulaire_commune_nom": "Rennes",
+        "titulaire_distance": 10,
+        "titulaire_typeIdentifiant": "SIRET",
+        "objet": "Objet test",
+        "dureeRestanteMois": 12,
+        "lieuExecution_code": "75001",
+        "sourceFile": "test.xml",
+        "sourceDataset": "test_dataset",
+        "datePublicationDonnees": datetime.date(2025, 1, 1),
+        "considerationsSociales": "",
+        "considerationsEnvironnementales": "",
+        "type": "Marché",
+        "acheteur_categorie": "Collectivité",
+        "titulaire_categorie": "PME",
+    }
+]
+_PARQUET_PATH = Path(os.path.abspath("tests/test.parquet"))
+_DB_PATH = Path(os.path.abspath("decp.duckdb"))
 
-@pytest.fixture(scope="session", autouse=True)
-def test_data():
-    data = [
-        {
-            "uid": "1",
-            "id": "1",
-            "acheteur_nom": "ACHETEUR 1",
-            "acheteur_id": "123",
-            "titulaire_nom": "TITULAIRE 1",
-            "titulaire_id": "345",
-            "montant": 10,
-            "dateNotification": datetime.date(2025, 1, 1),
-            "codeCPV": "71600000",
-            "donneesActuelles": True,
-            "acheteur_departement_code": "75",
-            "acheteur_departement_nom": "Paris",
-            "acheteur_commune_nom": "Paris",
-            "titulaire_departement_code": "35",
-            "titulaire_departement_nom": "Ille-et-Vilaine",
-            "titulaire_commune_nom": "Rennes",
-            "titulaire_distance": 10,
-            "titulaire_typeIdentifiant": "SIRET",
-            "objet": "Objet test",
-            "dureeRestanteMois": 12,
-            "lieuExecution_code": "75001",
-            "sourceFile": "test.xml",
-            "sourceDataset": "test_dataset",
-            "datePublicationDonnees": datetime.date(2025, 1, 1),
-            "considerationsSociales": "",
-            "considerationsEnvironnementales": "",
-            "type": "Marché",
-            "acheteur_categorie": "Collectivité",
-            "titulaire_categorie": "PME",
-        }
-    ]
-    parquet_path = Path(os.path.abspath("tests/test.parquet"))
-    db_path = parquet_path.parent / "decp.duckdb"
-    print(f"Writing test data to: {parquet_path}")
 
-    pl.DataFrame(data).write_parquet(parquet_path)
-
-    # Remove any stale DuckDB from a previous run so src.db rebuilds from
-    # the freshly-written parquet at import time.
-    for artifact in (db_path, db_path.with_suffix(".duckdb.tmp")):
+def _cleanup_db_artifacts() -> None:
+    for artifact in (
+        _DB_PATH,
+        _DB_PATH.with_suffix(".duckdb.tmp"),
+        _DB_PATH.with_suffix(".duckdb.lock"),
+    ):
         if artifact.exists():
             artifact.unlink()
 
-    yield str(parquet_path)
+
+# Runs at conftest import, before test modules import src.db (which builds the
+# DuckDB at import time). Guarantees the test parquet exists and the stale DB
+# from a previous `python run.py` is wiped so src.db rebuilds from test data.
+pl.DataFrame(_TEST_DATA).write_parquet(_PARQUET_PATH)
+_cleanup_db_artifacts()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def test_data():
+    yield str(_PARQUET_PATH)
+    # Teardown: remove the test DuckDB so the next `python run.py` rebuilds
+    # from decp_prod.parquet.
+    _cleanup_db_artifacts()
 
 
 def pytest_setup_options():

tests/test_db.py

@@ -141,6 +141,7 @@ def built_db(tmp_path, monkeypatch):
     )
     data.write_parquet(parquet_path)
     monkeypatch.setenv("DATA_FILE_PARQUET_PATH", str(parquet_path))
+    monkeypatch.setenv("DUCKDB_PATH", str(db_path))
 
     from src.db import build_database