From a2c20adb4ea2b515415b43287d1097fa4e0d8634 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 1 Jul 2026 13:46:44 +0200 Subject: [PATCH] =?UTF-8?q?perf:=20paginer/agr=C3=A9ger=20les=20pages=20ac?= =?UTF-8?q?heteur=20et=20titulaire=20c=C3=B4t=C3=A9=20DuckDB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Les pages acheteur/titulaire chargeaient l'intégralité des marchés d'une organisation dans un dcc.Store côté client (jusqu'à 16k lignes pour les plus gros acheteurs), envoyée sur le réseau à chaque interaction. Le tableau, le top 10 et l'histogramme des distances récupèrent maintenant leurs données via des requêtes DuckDB scopées (pagination/agrégation poussées en SQL), sur le modèle déjà utilisé par la page tableau. Corrige aussi le CLS des mêmes pages en réservant l'espace des conteneurs remplis par callback (carte, top 10, histogramme). Co-Authored-By: Claude Sonnet 5 --- src/figures.py | 2 +- src/pages/acheteur.py | 130 +++++++++++++++++++++++++---------------- src/pages/titulaire.py | 127 +++++++++++++++++++++++----------------- src/utils/table.py | 26 ++++++++- tests/test_main.py | 13 +---- 5 files changed, 179 insertions(+), 119 deletions(-) diff --git a/src/figures.py b/src/figures.py index 2801db3..b5bc04d 100644 --- a/src/figures.py +++ b/src/figures.py @@ -679,7 +679,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: ) fig.update_layout(bargap=0) - fig.update_layout(margin=dict(r=10, t=10)) + fig.update_layout(margin=dict(r=10, t=10), height=450) fig.update_xaxes( tickvals=[0, 1, 2, 3, 4], ticktext=["1", "10", "100", "1 000", "10 000"], diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 09d07d2..9ea4a38 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,5 +1,4 @@ import datetime -from typing import Any import dash_bootstrap_components as dbc import polars as pl @@ -15,7 +14,7 @@ from dash import ( register_page, ) -from src.db import query_marches, schema +from src.db import aggregate_marches, count_marches, query_marches, schema from src.figures import ( DataTable, get_distance_histogram, @@ -48,6 +47,17 @@ def get_title(acheteur_id: str | None = None) -> str: return "Marchés publics attribués | colibre" +def _acheteur_scope(pathname: str, ach_year: str | None) -> tuple[str, list]: + """WHERE SQL scopant les requêtes à cet acheteur (et éventuellement une année).""" + acheteur_siret = pathname.split("/")[-1] + where_sql = "acheteur_id = ?" + params: list = [acheteur_siret] + if ach_year and ach_year != "Toutes les années": + where_sql += ' AND YEAR("dateNotification") = ?' + params.append(int(ach_year)) + return where_sql, params + + register_page( __name__, path_template="/acheteurs/", @@ -75,7 +85,6 @@ DATATABLE = html.Div( ) layout = [ - dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Store(id="acheteur-hidden-columns", storage_type="local"), dcc.Store(id="filter-cleanup-trigger-acheteur"), dcc.Location(id="acheteur_url", refresh="callback-nav"), @@ -159,6 +168,7 @@ layout = [ dbc.Col( id="acheteur_map", width=4, + style={"minHeight": "300px"}, ), ], ), @@ -168,8 +178,13 @@ layout = [ className="marches_table", id="top10_titulaires", width=8, + style={"minHeight": "420px"}, + ), + dbc.Col( + id="acheteur-distance-histogram", + width=4, + style={"minHeight": "450px"}, ), - dbc.Col(id="acheteur-distance-histogram", width=4), ], ), ], @@ -310,48 +325,38 @@ def update_acheteur_infos(url): Output( component_id="acheteur_titulaires_differents", component_property="children" ), - Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), ) -def update_acheteur_stats(data): - dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) - if dff.height == 0: - dff = pl.DataFrame(schema=schema) - df_marches = dff.unique("id") - nb_marches = format_number(df_marches.height) - # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) - marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] - # + ", pour un total de ", html.Strong(somme_marches + " €")] - del df_marches +def update_acheteur_stats(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + agg = aggregate_marches( + "COUNT(*) AS n, COUNT(DISTINCT titulaire_id) AS nb_titulaires", + where_sql, + params, + ) + nb_marches = format_number(int(agg["n"][0])) if agg.height else "0" + nb_titulaires = format_number(int(agg["nb_titulaires"][0])) if agg.height else "0" - nb_titulaires = dff.unique("titulaire_id").height + marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] nb_titulaires = [ - html.Strong(format_number(nb_titulaires)), + html.Strong(nb_titulaires), " titulaires (SIRET) différents", ] - del dff return marches_attribues, nb_titulaires @callback( - Output(component_id="acheteur_data", component_property="data"), Output("btn-download-data-acheteur", "disabled"), Output("btn-download-data-acheteur", "children"), Output("btn-download-data-acheteur", "title"), Input(component_id="acheteur_url", component_property="pathname"), Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url, ach_year: str) -> tuple: - acheteur_siret = url.split("/")[-1] - lff = query_marches("acheteur_id = ?", (acheteur_siret,)).lazy() - if ach_year and ach_year != "Toutes les années": - ach_year = int(ach_year) - lff = lff.filter(pl.col("dateNotification").dt.year() == ach_year) - lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) - dff: pl.DataFrame = lff.collect(engine="streaming") - download_disabled, download_text, download_title = get_button_properties(dff.height) - data = dff.to_dicts() - return data, download_disabled, download_text, download_title +def update_download_button_acheteur(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + return get_button_properties(count_marches(where_sql, params)) @callback( @@ -364,8 +369,8 @@ def get_acheteur_marches_data(url, ach_year: str) -> tuple: Output("btn-download-filtered-data-acheteur", "children"), Output("btn-download-filtered-data-acheteur", "title"), Output("filter-cleanup-trigger-acheteur", "data"), - Input("acheteur_url", "href"), - Input("acheteur_data", "data"), + Input("acheteur_url", "pathname"), + Input("acheteur_year", "value"), Input("acheteur_datatable", "page_current"), Input("acheteur_datatable", "page_size"), Input("acheteur_datatable", "filter_query"), @@ -373,37 +378,59 @@ def get_acheteur_marches_data(url, ach_year: str) -> tuple: State("acheteur_datatable", "data_timestamp"), ) def get_last_marches_data( - href, data, page_current, page_size, filter_query, sort_by, data_timestamp + pathname, + ach_year, + page_current, + page_size, + filter_query, + sort_by, + data_timestamp, ) -> tuple: + where_sql, params = _acheteur_scope(pathname, ach_year) return prepare_table_data( - data, data_timestamp, filter_query, page_current, page_size, sort_by, "acheteur" + None, + data_timestamp, + filter_query, + page_current, + page_size, + sort_by, + "acheteur", + base_where_sql=where_sql, + base_params=params, ) @callback( Output(component_id="top10_titulaires", component_property="children"), - Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), ) -def get_top_titulaires(data): - table = get_top_org_table(data, "titulaire", ["titulaire_distance"]) +def get_top_titulaires(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + table = get_top_org_table( + query_marches(where_sql, params).lazy(), "titulaire", ["titulaire_distance"] + ) return make_card(fig=table, title="Top titulaires", lg=12, xl=12) @callback( Output("download-data-acheteur", "data"), Input("btn-download-data-acheteur", "n_clicks"), - State(component_id="acheteur_data", component_property="data"), - State(component_id="acheteur_nom", component_property="children"), + State(component_id="acheteur_url", component_property="pathname"), State(component_id="acheteur_year", component_property="value"), + State(component_id="acheteur_nom", component_property="children"), prevent_initial_call=True, ) def download_acheteur_data( n_clicks, - data: list[dict[str, Any]], - acheteur_nom: str, + pathname: str, annee: str, + acheteur_nom: str, ): - df_to_download = pl.DataFrame(data) + where_sql, params = _acheteur_scope(pathname, annee) + df_to_download = query_marches( + where_sql, params, order_by='"dateNotification" DESC, uid DESC' + ) def to_bytes(buffer): write_styled_excel( @@ -418,8 +445,9 @@ def download_acheteur_data( @callback( Output("acheteur-download-filtered-data", "data"), - State("acheteur_data", "data"), Input("btn-download-filtered-data-acheteur", "n_clicks"), + State("acheteur_url", "pathname"), + State("acheteur_year", "value"), State("acheteur_nom", "children"), State("acheteur_datatable", "filter_query"), State("acheteur_datatable", "sort_by"), @@ -427,16 +455,16 @@ def download_acheteur_data( prevent_initial_call=True, ) def download_filtered_acheteur_data( - data, n_clicks, + pathname, + ach_year, acheteur_nom, filter_query, sort_by, hidden_columns: list | None = None, ): - lff: pl.LazyFrame = pl.LazyFrame( - data - ) # start from the full acheteur data, not from paginated table data + where_sql, params = _acheteur_scope(pathname, ach_year) + lff: pl.LazyFrame = query_marches(where_sql, params).lazy() # Les colonnes masquées sont supprimées if hidden_columns: @@ -535,10 +563,12 @@ def reset_view(n_clicks): @callback( Output("acheteur-distance-histogram", "children"), - Input("acheteur_data", "data"), + Input("acheteur_url", "pathname"), + Input("acheteur_year", "value"), ) -def update_acheteur_distance_histogram(data): - lff = pl.LazyFrame(data, strict=False, infer_schema_length=5000) +def update_acheteur_distance_histogram(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + lff = query_marches(where_sql, params).lazy() fig = get_distance_histogram(lff) return make_card( title="Distance acheteur–titulaire", diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 6ce3eb1..d84ec4e 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -1,5 +1,4 @@ import datetime -from typing import Any import dash_bootstrap_components as dbc import polars as pl @@ -15,7 +14,7 @@ from dash import ( register_page, ) -from src.db import query_marches, schema +from src.db import aggregate_marches, count_marches, query_marches, schema from src.figures import ( DataTable, get_distance_histogram, @@ -47,6 +46,17 @@ def get_title(titulaire_id: str = None) -> str: return "Marchés publics remportés | colibre" +def _titulaire_scope(pathname: str, titulaire_year: str | None) -> tuple[str, list]: + """WHERE SQL scopant les requêtes à ce titulaire (et éventuellement une année).""" + titulaire_siret = pathname.split("/")[-1] + where_sql = "titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'" + params: list = [titulaire_siret] + if titulaire_year and titulaire_year != "Toutes les années": + where_sql += ' AND YEAR("dateNotification") = ?' + params.append(int(titulaire_year)) + return where_sql, params + + register_page( __name__, path_template="/titulaires/", @@ -74,7 +84,6 @@ DATATABLE = html.Div( ) layout = [ - dcc.Store(id="titulaire_data", storage_type="memory"), dcc.Store(id="titulaire-hidden-columns", storage_type="local"), dcc.Store(id="filter-cleanup-trigger-titulaire"), dcc.Location(id="titulaire_url", refresh="callback-nav"), @@ -167,6 +176,7 @@ layout = [ dbc.Col( id="titulaire_map", width=4, + style={"minHeight": "300px"}, ), ], ), @@ -179,12 +189,17 @@ layout = [ html.Div( className="marches_table", id="top10_acheteurs", + style={"minHeight": "420px"}, ), ], ), width=8, ), - dbc.Col(id="titulaire-distance-histogram", width=4), + dbc.Col( + id="titulaire-distance-histogram", + width=4, + style={"minHeight": "450px"}, + ), ], ), ], @@ -334,26 +349,25 @@ def update_titulaire_infos(url): Output( component_id="titulaire_acheteurs_differents", component_property="children" ), - Input(component_id="titulaire_data", component_property="data"), + Input(component_id="titulaire_url", component_property="pathname"), + Input(component_id="titulaire_year", component_property="value"), ) -def update_titulaire_stats(data): - dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) - if dff.height == 0: - nb_marches = 0 - nb_acheteurs = 0 - else: - df_marches = dff.unique("uid") - nb_marches = format_number(df_marches.height) - nb_acheteurs = dff.unique("acheteur_id").height +def update_titulaire_stats(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + agg = aggregate_marches( + "COUNT(DISTINCT uid) AS n, COUNT(DISTINCT acheteur_id) AS nb_acheteurs", + where_sql, + params, + ) + nb_marches = format_number(int(agg["n"][0])) if agg.height else "0" + nb_acheteurs = format_number(int(agg["nb_acheteurs"][0])) if agg.height else "0" texte_marches_remportes = [ html.Strong(nb_marches), " marchés et accord-cadres remportés", ] - # + ", pour un total de ", html.Strong(somme_marches + " €")] - texte_nb_acheteurs = [ - html.Strong(format_number(nb_acheteurs)), + html.Strong(nb_acheteurs), " acheteurs (SIRET) différents", ] @@ -361,29 +375,15 @@ def update_titulaire_stats(data): @callback( - Output(component_id="titulaire_data", component_property="data"), Output("btn-download-data-titulaire", "disabled"), Output("btn-download-data-titulaire", "children"), Output("btn-download-data-titulaire", "title"), Input(component_id="titulaire_url", component_property="pathname"), Input(component_id="titulaire_year", component_property="value"), ) -def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: - titulaire_siret = url.split("/")[-1] - lff = query_marches( - "titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'", - (titulaire_siret,), - ).lazy() - if titulaire_year and titulaire_year != "Toutes les années": - lff = lff.filter( - pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year) - ) - lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) - lff = lff.fill_null("") - dff: pl.DataFrame = lff.collect(engine="streaming") - download_disabled, download_text, download_title = get_button_properties(dff.height) - data = dff.to_dicts() - return data, download_disabled, download_text, download_title +def update_download_button_titulaire(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + return get_button_properties(count_marches(where_sql, params)) @callback( @@ -396,8 +396,8 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: Output("btn-download-filtered-data-titulaire", "children"), Output("btn-download-filtered-data-titulaire", "title"), Output("filter-cleanup-trigger-titulaire", "data"), - Input(component_id="titulaire_url", component_property="href"), - Input("titulaire_data", "data"), + Input("titulaire_url", "pathname"), + Input("titulaire_year", "value"), Input("titulaire_datatable", "page_current"), Input("titulaire_datatable", "page_size"), Input("titulaire_datatable", "filter_query"), @@ -405,42 +405,58 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: State("titulaire_datatable", "data_timestamp"), ) def get_last_marches_data( - href, data, page_current, page_size, filter_query, sort_by, data_timestamp + pathname, + titulaire_year, + page_current, + page_size, + filter_query, + sort_by, + data_timestamp, ) -> list[dict]: + where_sql, params = _titulaire_scope(pathname, titulaire_year) return prepare_table_data( - data, + None, data_timestamp, filter_query, page_current, page_size, sort_by, "titulaire", + base_where_sql=where_sql, + base_params=params, ) @callback( Output(component_id="top10_acheteurs", component_property="children"), - Input(component_id="titulaire_data", component_property="data"), + Input(component_id="titulaire_url", component_property="pathname"), + Input(component_id="titulaire_year", component_property="value"), ) -def get_top_acheteurs(data): - return get_top_org_table(data, "acheteur", ["titulaire_distance"]) +def get_top_acheteurs(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + return get_top_org_table( + query_marches(where_sql, params).lazy(), "acheteur", ["titulaire_distance"] + ) @callback( Output("download-data-titulaire", "data"), Input("btn-download-data-titulaire", "n_clicks"), - State(component_id="titulaire_data", component_property="data"), - State(component_id="titulaire_nom", component_property="children"), + State(component_id="titulaire_url", component_property="pathname"), State(component_id="titulaire_year", component_property="value"), + State(component_id="titulaire_nom", component_property="children"), prevent_initial_call=True, ) def download_titulaire_data( n_clicks, - data: list[dict[str, Any]], - titulaire_nom: str, + pathname: str, annee: str, + titulaire_nom: str, ): - df_to_download = pl.DataFrame(data) + where_sql, params = _titulaire_scope(pathname, annee) + df_to_download = query_marches( + where_sql, params, order_by='"dateNotification" DESC, uid DESC' + ).fill_null("") def to_bytes(buffer): write_styled_excel( @@ -455,8 +471,9 @@ def download_titulaire_data( @callback( Output("titulaire-download-filtered-data", "data"), - State("titulaire_data", "data"), Input("btn-download-filtered-data-titulaire", "n_clicks"), + State("titulaire_url", "pathname"), + State("titulaire_year", "value"), State("titulaire_nom", "children"), State("titulaire_datatable", "filter_query"), State("titulaire_datatable", "sort_by"), @@ -464,16 +481,16 @@ def download_titulaire_data( prevent_initial_call=True, ) def download_filtered_titulaire_data( - data, n_clicks, + pathname, + titulaire_year, titulaire_nom, filter_query, sort_by, hidden_columns: list | None = None, ): - lff: pl.LazyFrame = pl.LazyFrame( - data - ) # start from the full titulaire data, not from paginated table data + where_sql, params = _titulaire_scope(pathname, titulaire_year) + lff: pl.LazyFrame = query_marches(where_sql, params).lazy() # Les colonnes masquées sont supprimées if hidden_columns: @@ -572,10 +589,12 @@ def reset_view(n_clicks): @callback( Output("titulaire-distance-histogram", "children"), - Input("titulaire_data", "data"), + Input("titulaire_url", "pathname"), + Input("titulaire_year", "value"), ) -def update_titulaire_distance_histogram(data): - lff = pl.LazyFrame(data) +def update_titulaire_distance_histogram(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + lff = query_marches(where_sql, params).lazy() if "titulaire_distance" in lff.collect_schema().names(): lff = lff.with_columns( pl.col("titulaire_distance").cast(pl.Float64, strict=False) diff --git a/src/utils/table.py b/src/utils/table.py index 0298012..1d7681a 100644 --- a/src/utils/table.py +++ b/src/utils/table.py @@ -427,9 +427,14 @@ def _fetch_page_sql( sort_by_key: tuple, page_current: int, page_size: int, + base_where_sql: str = "TRUE", + base_params: tuple = (), ) -> tuple[pl.DataFrame, int, int]: """Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule. + `base_where_sql`/`base_params` permettent de scoper la requête (ex : un + acheteur ou un titulaire précis) en plus du filtre saisi dans la table. + Retourne (page_dataframe_post_traitée, total_count, total_unique_count). """ # Import local pour éviter une dépendance circulaire @@ -441,7 +446,9 @@ def _fetch_page_sql( f"page={page_current} size={page_size}" ) - where_sql, params = filter_query_to_sql(filter_query or "", schema) + filter_where_sql, filter_params = filter_query_to_sql(filter_query or "", schema) + where_sql = f"({base_where_sql}) AND ({filter_where_sql})" + params = [*base_params, *filter_params] sort_by_dash = [ {"column_id": col, "direction": direction} for col, direction in sort_by_key @@ -464,7 +471,15 @@ def _fetch_page_sql( def prepare_table_data( - data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table + data, + data_timestamp, + filter_query, + page_current, + page_size, + sort_by, + source_table, + base_where_sql: str = "TRUE", + base_params: tuple = (), ): """ Fonction de préparation des données pour les datatables, afin de permettre une gestion fine des logiques, @@ -476,6 +491,8 @@ def prepare_table_data( :param page_size: :param sort_by: :param source_table: + :param base_where_sql: scope SQL additionnel (ex : un acheteur/titulaire précis) + :param base_params: paramètres liés à base_where_sql :return: """ logger.debug(" + + + + + + + + + + + + + + + + + + ") @@ -486,13 +503,16 @@ def prepare_table_data( trigger_cleanup = no_update if source_table == "tableau" else str(uuid.uuid4()) if data is None: - # Probablement car il s'agit de la page Tableau + # Chemin rapide SQL : tableau, acheteur, titulaire (scope éventuel via + # base_where_sql/base_params) sort_by_key = normalize_sort_by(sort_by) dff, height, total_unique = _fetch_page_sql( filter_query=filter_query, sort_by_key=sort_by_key, page_current=page_current, page_size=page_size, + base_where_sql=base_where_sql, + base_params=tuple(base_params), ) else: if isinstance(data, list): diff --git a/tests/test_main.py b/tests/test_main.py index f88571f..b458dc3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -98,12 +98,10 @@ def test_003_tableau_download(dash_duo: DashComposite): # Juste pour instancier l'app print(app.server.name) - dicts = pl.read_parquet("tests/test.parquet").to_dicts() - outputs = [ download_data(1, "", [], None), - download_acheteur_data(1, dicts, "123", "2025"), - download_titulaire_data(1, dicts, "345", "2025"), + download_acheteur_data(1, "/acheteurs/123", "2025", "ACHETEUR 1"), + download_titulaire_data(1, "/titulaires/345", "2025", "TITULAIRE 1"), ] for output in outputs: assert isinstance(output, dict) @@ -118,8 +116,6 @@ def test_003_tableau_download(dash_duo: DashComposite): def test_004_add_links_observatoire_acheteur(): - import polars as pl - from src.utils.table import add_links dff = pl.DataFrame( @@ -143,8 +139,6 @@ def test_004_add_links_observatoire_acheteur(): def test_005_add_links_observatoire_titulaire(): - import polars as pl - from src.utils.table import add_links dff = pl.DataFrame( @@ -323,7 +317,6 @@ def test_011_observatoire_multi_param_url(dash_duo: DashComposite): def test_012_get_distance_histogram_returns_graph(): - import polars as pl from dash import dcc from src.figures import get_distance_histogram @@ -334,7 +327,6 @@ def test_012_get_distance_histogram_returns_graph(): def test_013_get_distance_histogram_handles_nulls(): - import polars as pl from dash import dcc from src.figures import get_distance_histogram @@ -345,7 +337,6 @@ def test_013_get_distance_histogram_handles_nulls(): def test_014_get_distance_histogram_all_nulls(): - import polars as pl from dash import dcc from src.figures import get_distance_histogram