From 3532a9c38156b575a84ea0d14827ce146ad8c2c1 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 22 Apr 2026 20:43:38 +0200 Subject: [PATCH] Path de duckdb configurable, correction des tests #72 --- .template.env | 1 + pyproject.toml | 1 + src/db.py | 2 +- tests/conftest.py | 99 ++++++++++++++++++++++++++--------------------- tests/test_db.py | 1 + 5 files changed, 59 insertions(+), 45 deletions(-) diff --git a/.template.env b/.template.env index 67df0ba..6ac9a91 100644 --- a/.template.env +++ b/.template.env @@ -1,4 +1,5 @@ DATA_FILE_PARQUET_PATH=https://www.data.gouv.fr/fr/datasets/r/11cea8e8-df3e-4ed1-932b-781e2635e432 +DUCKDB_PATH=./decp.duckdb PORT=8050 DEVELOPMENT=True SOURCE_STATS_CSV_PATH="https://www.data.gouv.fr/api/1/datasets/r/8ded94de-3b80-4840-a5bb-7faad1c9c234" diff --git a/pyproject.toml b/pyproject.toml index 4c8748c..212d0bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ testpaths = ["tests"] env = [ "DATA_FILE_PARQUET_PATH=tests/test.parquet", "DEVELOPMENT=true", + "REBUILD_DUCKDB=true", "DATA_SCHEMA_PATH=/home/colin/git/decp-processing/dist/schema.json", ] addopts = "-p no:warnings" diff --git a/src/db.py b/src/db.py index 9290e48..f9ffd39 100644 --- a/src/db.py +++ b/src/db.py @@ -110,7 +110,7 @@ def build_database(db_path: Path, parquet_path: Path) -> None: def _ensure_database() -> Path: - db_path = Path("./decp.duckdb") + db_path = Path(os.getenv("DUCKDB_PATH", "./decp.duckdb")) parquet_path = Path(os.getenv("DATA_FILE_PARQUET_PATH")) lock_path = db_path.with_suffix(".duckdb.lock") diff --git a/tests/conftest.py b/tests/conftest.py index df103aa..f8d702f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,55 +6,66 @@ import polars as pl import pytest from selenium.webdriver.chrome.options import Options +_TEST_DATA = [ + { + "uid": "1", + "id": "1", + "acheteur_nom": "ACHETEUR 1", + "acheteur_id": "123", + "titulaire_nom": "TITULAIRE 1", + "titulaire_id": "345", + "montant": 10, + "dateNotification": datetime.date(2025, 1, 1), + "codeCPV": "71600000", + "donneesActuelles": True, + "acheteur_departement_code": "75", + "acheteur_departement_nom": "Paris", + "acheteur_commune_nom": "Paris", + "titulaire_departement_code": "35", + "titulaire_departement_nom": "Ille-et-Vilaine", + "titulaire_commune_nom": "Rennes", + "titulaire_distance": 10, + "titulaire_typeIdentifiant": "SIRET", + "objet": "Objet test", + "dureeRestanteMois": 12, + "lieuExecution_code": "75001", + "sourceFile": "test.xml", + "sourceDataset": "test_dataset", + "datePublicationDonnees": datetime.date(2025, 1, 1), + "considerationsSociales": "", + "considerationsEnvironnementales": "", + "type": "Marché", + "acheteur_categorie": "Collectivité", + "titulaire_categorie": "PME", + } +] +_PARQUET_PATH = Path(os.path.abspath("tests/test.parquet")) +_DB_PATH = Path(os.path.abspath("decp.duckdb")) -@pytest.fixture(scope="session", autouse=True) -def test_data(): - data = [ - { - "uid": "1", - "id": "1", - "acheteur_nom": "ACHETEUR 1", - "acheteur_id": "123", - "titulaire_nom": "TITULAIRE 1", - "titulaire_id": "345", - "montant": 10, - "dateNotification": datetime.date(2025, 1, 1), - "codeCPV": "71600000", - "donneesActuelles": True, - "acheteur_departement_code": "75", - "acheteur_departement_nom": "Paris", - "acheteur_commune_nom": "Paris", - "titulaire_departement_code": "35", - "titulaire_departement_nom": "Ille-et-Vilaine", - "titulaire_commune_nom": "Rennes", - "titulaire_distance": 10, - "titulaire_typeIdentifiant": "SIRET", - "objet": "Objet test", - "dureeRestanteMois": 12, - "lieuExecution_code": "75001", - "sourceFile": "test.xml", - "sourceDataset": "test_dataset", - "datePublicationDonnees": datetime.date(2025, 1, 1), - "considerationsSociales": "", - "considerationsEnvironnementales": "", - "type": "Marché", - "acheteur_categorie": "Collectivité", - "titulaire_categorie": "PME", - } - ] - parquet_path = Path(os.path.abspath("tests/test.parquet")) - db_path = parquet_path.parent / "decp.duckdb" - print(f"Writing test data to: {parquet_path}") - pl.DataFrame(data).write_parquet(parquet_path) - - # Remove any stale DuckDB from a previous run so src.db rebuilds from - # the freshly-written parquet at import time. - for artifact in (db_path, db_path.with_suffix(".duckdb.tmp")): +def _cleanup_db_artifacts() -> None: + for artifact in ( + _DB_PATH, + _DB_PATH.with_suffix(".duckdb.tmp"), + _DB_PATH.with_suffix(".duckdb.lock"), + ): if artifact.exists(): artifact.unlink() - yield str(parquet_path) + +# Runs at conftest import, before test modules import src.db (which builds the +# DuckDB at import time). Guarantees the test parquet exists and the stale DB +# from a previous `python run.py` is wiped so src.db rebuilds from test data. +pl.DataFrame(_TEST_DATA).write_parquet(_PARQUET_PATH) +_cleanup_db_artifacts() + + +@pytest.fixture(scope="session", autouse=True) +def test_data(): + yield str(_PARQUET_PATH) + # Teardown: remove the test DuckDB so the next `python run.py` rebuilds + # from decp_prod.parquet. + _cleanup_db_artifacts() def pytest_setup_options(): diff --git a/tests/test_db.py b/tests/test_db.py index 0ca9e36..bba7de5 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -141,6 +141,7 @@ def built_db(tmp_path, monkeypatch): ) data.write_parquet(parquet_path) monkeypatch.setenv("DATA_FILE_PARQUET_PATH", str(parquet_path)) + monkeypatch.setenv("DUCKDB_PATH", str(db_path)) from src.db import build_database