colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

refactor(observatoire): prepare_dashboard_data utilise DuckDB (#72)

Browse Source
This commit is contained in:
Colin Maudry
2026-04-22 23:55:45 +02:00
parent 6e670c97c9
commit 0777153c82
2 changed files with 10 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/data.py
+10 -109
View File
@@ -2,12 +2,11 @@ import json
import logging import logging
import os import os
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime, timedelta
import polars as pl import polars as pl
from httpx import HTTPError, get from httpx import HTTPError, get
from src.db import get_cursor, schema from src.db import get_cursor, query_marches, schema
from src.utils import logger from src.utils import logger
logging.getLogger("httpx").setLevel("WARNING") logging.getLogger("httpx").setLevel("WARNING")
@@ -83,115 +82,17 @@ def get_data_schema() -> dict:
return new_schema return new_schema
def prepare_dashboard_data( def prepare_dashboard_data(**filter_params) -> pl.DataFrame:
lff: pl.LazyFrame, """Exécute la requête DuckDB filtrée pour le tableau de bord.
dashboard_year=None,
dashboard_acheteur_id=None,
dashboard_acheteur_categorie=None,
dashboard_acheteur_departement_code=None,
dashboard_titulaire_id=None,
dashboard_titulaire_categorie=None,
dashboard_titulaire_departement_code=None,
dashboard_marche_type=None,
dashboard_marche_objet=None,
dashboard_marche_code_cpv=None,
dashboard_marche_considerations_sociales=None,
dashboard_marche_considerations_environnementales=None,
dashboard_marche_techniques=None,
dashboard_marche_innovant=None,
dashboard_marche_sous_traitance_declaree=None,
dashboard_montant_min=None,
dashboard_montant_max=None,
) -> pl.LazyFrame:
if dashboard_year:
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
else:
lff = lff.filter(
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
)
if dashboard_acheteur_id: Retourne une pl.DataFrame matérialisée uniquement pour le sous-ensemble
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) correspondant aux filtres. Les appelants qui ont besoin d'une LazyFrame
else: appellent `.lazy()` sur le résultat.
if dashboard_acheteur_categorie: """
lff = lff.filter( from src.utils.table_sql import dashboard_filters_to_sql
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
)
if dashboard_acheteur_departement_code:
lff = lff.filter(
pl.col("acheteur_departement_code").is_in(
dashboard_acheteur_departement_code
)
)
if dashboard_titulaire_id: where_sql, params = dashboard_filters_to_sql(**filter_params)
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) return query_marches(where_sql=where_sql, params=params)
else:
if dashboard_titulaire_categorie:
lff = lff.filter(
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
)
if dashboard_titulaire_departement_code:
lff = lff.filter(
pl.col("titulaire_departement_code").is_in(
dashboard_titulaire_departement_code
)
)
if dashboard_marche_type:
lff = lff.filter(pl.col("type") == dashboard_marche_type)
if dashboard_marche_objet:
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
if dashboard_marche_code_cpv:
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
if (
dashboard_marche_sous_traitance_declaree
and dashboard_marche_sous_traitance_declaree != "all"
):
lff = lff.filter(
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
)
if dashboard_marche_techniques:
lff = lff.filter(
pl.col("techniques")
.str.split(", ")
.list.set_intersection(dashboard_marche_techniques)
.list.len()
> 0
)
if dashboard_marche_considerations_sociales:
lff = lff.filter(
pl.col("considerationsSociales")
.str.split(", ")
.list.set_intersection(dashboard_marche_considerations_sociales)
.list.len()
> 0
)
if dashboard_marche_considerations_environnementales:
lff = lff.filter(
pl.col("considerationsEnvironnementales")
.str.split(", ")
.list.set_intersection(dashboard_marche_considerations_environnementales)
.list.len()
> 0
)
if dashboard_montant_min is not None:
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
if dashboard_montant_max is not None:
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
return lff
def build_org_frame(org_type: str) -> pl.DataFrame: def build_org_frame(org_type: str) -> pl.DataFrame:
tests/test_main.py
-41
View File
@@ -215,47 +215,6 @@ def test_008_search_to_observatoire(dash_duo: DashComposite):
) )
def test_010_observatoire_montant_filter():
import datetime
from src.utils.data import prepare_dashboard_data
data = pl.DataFrame(
{
"uid": ["1", "2", "3"],
"montant": [100.0, 500.0, 1000.0],
"dateNotification": [datetime.date(2025, 1, 1)] * 3,
}
)
def apply(min_val=None, max_val=None):
return prepare_dashboard_data(
data.lazy(),
dashboard_year="2025",
dashboard_acheteur_id=None,
dashboard_acheteur_categorie=None,
dashboard_acheteur_departement_code=None,
dashboard_titulaire_id=None,
dashboard_titulaire_categorie=None,
dashboard_titulaire_departement_code=None,
dashboard_marche_type=None,
dashboard_marche_objet=None,
dashboard_marche_code_cpv=None,
dashboard_marche_considerations_sociales=None,
dashboard_marche_considerations_environnementales=None,
dashboard_marche_techniques=None,
dashboard_marche_innovant=None,
dashboard_marche_sous_traitance_declaree=None,
dashboard_montant_min=min_val,
dashboard_montant_max=max_val,
).collect()
assert apply().height == 3
assert apply(min_val=400).height == 2 # 500, 1000
assert apply(max_val=500).height == 2 # 100, 500
assert apply(min_val=200, max_val=600).height == 1 # 500 only
def test_009_observatoire_filter_persistence(dash_duo: DashComposite): def test_009_observatoire_filter_persistence(dash_duo: DashComposite):
import time import time