From 0777153c823089e153a23807676f40f67a013106 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 22 Apr 2026 23:55:45 +0200 Subject: [PATCH] refactor(observatoire): prepare_dashboard_data utilise DuckDB (#72) --- src/utils/data.py | 119 ++++----------------------------------------- tests/test_main.py | 41 ---------------- 2 files changed, 10 insertions(+), 150 deletions(-) diff --git a/src/utils/data.py b/src/utils/data.py index e4817be..cd54f40 100644 --- a/src/utils/data.py +++ b/src/utils/data.py @@ -2,12 +2,11 @@ import json import logging import os from collections import OrderedDict -from datetime import datetime, timedelta import polars as pl from httpx import HTTPError, get -from src.db import get_cursor, schema +from src.db import get_cursor, query_marches, schema from src.utils import logger logging.getLogger("httpx").setLevel("WARNING") @@ -83,115 +82,17 @@ def get_data_schema() -> dict: return new_schema -def prepare_dashboard_data( - lff: pl.LazyFrame, - dashboard_year=None, - dashboard_acheteur_id=None, - dashboard_acheteur_categorie=None, - dashboard_acheteur_departement_code=None, - dashboard_titulaire_id=None, - dashboard_titulaire_categorie=None, - dashboard_titulaire_departement_code=None, - dashboard_marche_type=None, - dashboard_marche_objet=None, - dashboard_marche_code_cpv=None, - dashboard_marche_considerations_sociales=None, - dashboard_marche_considerations_environnementales=None, - dashboard_marche_techniques=None, - dashboard_marche_innovant=None, - dashboard_marche_sous_traitance_declaree=None, - dashboard_montant_min=None, - dashboard_montant_max=None, -) -> pl.LazyFrame: - if dashboard_year: - lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) - else: - lff = lff.filter( - pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) - ) +def prepare_dashboard_data(**filter_params) -> pl.DataFrame: + """Exécute la requête DuckDB filtrée pour le tableau de bord. - if dashboard_acheteur_id: - lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) - else: - if dashboard_acheteur_categorie: - lff = lff.filter( - pl.col("acheteur_categorie") == dashboard_acheteur_categorie - ) - if dashboard_acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in( - dashboard_acheteur_departement_code - ) - ) + Retourne une pl.DataFrame matérialisée uniquement pour le sous-ensemble + correspondant aux filtres. Les appelants qui ont besoin d'une LazyFrame + appellent `.lazy()` sur le résultat. + """ + from src.utils.table_sql import dashboard_filters_to_sql - if dashboard_titulaire_id: - lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) - else: - if dashboard_titulaire_categorie: - lff = lff.filter( - pl.col("titulaire_categorie") == dashboard_titulaire_categorie - ) - if dashboard_titulaire_departement_code: - lff = lff.filter( - pl.col("titulaire_departement_code").is_in( - dashboard_titulaire_departement_code - ) - ) - - if dashboard_marche_type: - lff = lff.filter(pl.col("type") == dashboard_marche_type) - - if dashboard_marche_objet: - lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}")) - - if dashboard_marche_code_cpv: - lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv)) - - if dashboard_marche_innovant and dashboard_marche_innovant != "all": - lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant) - - if ( - dashboard_marche_sous_traitance_declaree - and dashboard_marche_sous_traitance_declaree != "all" - ): - lff = lff.filter( - pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree - ) - - if dashboard_marche_techniques: - lff = lff.filter( - pl.col("techniques") - .str.split(", ") - .list.set_intersection(dashboard_marche_techniques) - .list.len() - > 0 - ) - - if dashboard_marche_considerations_sociales: - lff = lff.filter( - pl.col("considerationsSociales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_sociales) - .list.len() - > 0 - ) - - if dashboard_marche_considerations_environnementales: - lff = lff.filter( - pl.col("considerationsEnvironnementales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_environnementales) - .list.len() - > 0 - ) - - if dashboard_montant_min is not None: - lff = lff.filter(pl.col("montant") >= dashboard_montant_min) - - if dashboard_montant_max is not None: - lff = lff.filter(pl.col("montant") <= dashboard_montant_max) - - return lff + where_sql, params = dashboard_filters_to_sql(**filter_params) + return query_marches(where_sql=where_sql, params=params) def build_org_frame(org_type: str) -> pl.DataFrame: diff --git a/tests/test_main.py b/tests/test_main.py index 7f55a02..ae9b36c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -215,47 +215,6 @@ def test_008_search_to_observatoire(dash_duo: DashComposite): ) -def test_010_observatoire_montant_filter(): - import datetime - - from src.utils.data import prepare_dashboard_data - - data = pl.DataFrame( - { - "uid": ["1", "2", "3"], - "montant": [100.0, 500.0, 1000.0], - "dateNotification": [datetime.date(2025, 1, 1)] * 3, - } - ) - - def apply(min_val=None, max_val=None): - return prepare_dashboard_data( - data.lazy(), - dashboard_year="2025", - dashboard_acheteur_id=None, - dashboard_acheteur_categorie=None, - dashboard_acheteur_departement_code=None, - dashboard_titulaire_id=None, - dashboard_titulaire_categorie=None, - dashboard_titulaire_departement_code=None, - dashboard_marche_type=None, - dashboard_marche_objet=None, - dashboard_marche_code_cpv=None, - dashboard_marche_considerations_sociales=None, - dashboard_marche_considerations_environnementales=None, - dashboard_marche_techniques=None, - dashboard_marche_innovant=None, - dashboard_marche_sous_traitance_declaree=None, - dashboard_montant_min=min_val, - dashboard_montant_max=max_val, - ).collect() - - assert apply().height == 3 - assert apply(min_val=400).height == 2 # 500, 1000 - assert apply(max_val=500).height == 2 # 100, 500 - assert apply(min_val=200, max_val=600).height == 1 # 500 only - - def test_009_observatoire_filter_persistence(dash_duo: DashComposite): import time