diff --git a/src/figures.py b/src/figures.py index 2801db3..b5bc04d 100644 --- a/src/figures.py +++ b/src/figures.py @@ -679,7 +679,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: ) fig.update_layout(bargap=0) - fig.update_layout(margin=dict(r=10, t=10)) + fig.update_layout(margin=dict(r=10, t=10), height=450) fig.update_xaxes( tickvals=[0, 1, 2, 3, 4], ticktext=["1", "10", "100", "1 000", "10 000"], diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 09d07d2..9ea4a38 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,5 +1,4 @@ import datetime -from typing import Any import dash_bootstrap_components as dbc import polars as pl @@ -15,7 +14,7 @@ from dash import ( register_page, ) -from src.db import query_marches, schema +from src.db import aggregate_marches, count_marches, query_marches, schema from src.figures import ( DataTable, get_distance_histogram, @@ -48,6 +47,17 @@ def get_title(acheteur_id: str | None = None) -> str: return "Marchés publics attribués | colibre" +def _acheteur_scope(pathname: str, ach_year: str | None) -> tuple[str, list]: + """WHERE SQL scopant les requêtes à cet acheteur (et éventuellement une année).""" + acheteur_siret = pathname.split("/")[-1] + where_sql = "acheteur_id = ?" + params: list = [acheteur_siret] + if ach_year and ach_year != "Toutes les années": + where_sql += ' AND YEAR("dateNotification") = ?' + params.append(int(ach_year)) + return where_sql, params + + register_page( __name__, path_template="/acheteurs/", @@ -75,7 +85,6 @@ DATATABLE = html.Div( ) layout = [ - dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Store(id="acheteur-hidden-columns", storage_type="local"), dcc.Store(id="filter-cleanup-trigger-acheteur"), dcc.Location(id="acheteur_url", refresh="callback-nav"), @@ -159,6 +168,7 @@ layout = [ dbc.Col( id="acheteur_map", width=4, + style={"minHeight": "300px"}, ), ], ), @@ -168,8 +178,13 @@ layout = [ className="marches_table", id="top10_titulaires", width=8, + style={"minHeight": "420px"}, + ), + dbc.Col( + id="acheteur-distance-histogram", + width=4, + style={"minHeight": "450px"}, ), - dbc.Col(id="acheteur-distance-histogram", width=4), ], ), ], @@ -310,48 +325,38 @@ def update_acheteur_infos(url): Output( component_id="acheteur_titulaires_differents", component_property="children" ), - Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), ) -def update_acheteur_stats(data): - dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) - if dff.height == 0: - dff = pl.DataFrame(schema=schema) - df_marches = dff.unique("id") - nb_marches = format_number(df_marches.height) - # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) - marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] - # + ", pour un total de ", html.Strong(somme_marches + " €")] - del df_marches +def update_acheteur_stats(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + agg = aggregate_marches( + "COUNT(*) AS n, COUNT(DISTINCT titulaire_id) AS nb_titulaires", + where_sql, + params, + ) + nb_marches = format_number(int(agg["n"][0])) if agg.height else "0" + nb_titulaires = format_number(int(agg["nb_titulaires"][0])) if agg.height else "0" - nb_titulaires = dff.unique("titulaire_id").height + marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] nb_titulaires = [ - html.Strong(format_number(nb_titulaires)), + html.Strong(nb_titulaires), " titulaires (SIRET) différents", ] - del dff return marches_attribues, nb_titulaires @callback( - Output(component_id="acheteur_data", component_property="data"), Output("btn-download-data-acheteur", "disabled"), Output("btn-download-data-acheteur", "children"), Output("btn-download-data-acheteur", "title"), Input(component_id="acheteur_url", component_property="pathname"), Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url, ach_year: str) -> tuple: - acheteur_siret = url.split("/")[-1] - lff = query_marches("acheteur_id = ?", (acheteur_siret,)).lazy() - if ach_year and ach_year != "Toutes les années": - ach_year = int(ach_year) - lff = lff.filter(pl.col("dateNotification").dt.year() == ach_year) - lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) - dff: pl.DataFrame = lff.collect(engine="streaming") - download_disabled, download_text, download_title = get_button_properties(dff.height) - data = dff.to_dicts() - return data, download_disabled, download_text, download_title +def update_download_button_acheteur(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + return get_button_properties(count_marches(where_sql, params)) @callback( @@ -364,8 +369,8 @@ def get_acheteur_marches_data(url, ach_year: str) -> tuple: Output("btn-download-filtered-data-acheteur", "children"), Output("btn-download-filtered-data-acheteur", "title"), Output("filter-cleanup-trigger-acheteur", "data"), - Input("acheteur_url", "href"), - Input("acheteur_data", "data"), + Input("acheteur_url", "pathname"), + Input("acheteur_year", "value"), Input("acheteur_datatable", "page_current"), Input("acheteur_datatable", "page_size"), Input("acheteur_datatable", "filter_query"), @@ -373,37 +378,59 @@ def get_acheteur_marches_data(url, ach_year: str) -> tuple: State("acheteur_datatable", "data_timestamp"), ) def get_last_marches_data( - href, data, page_current, page_size, filter_query, sort_by, data_timestamp + pathname, + ach_year, + page_current, + page_size, + filter_query, + sort_by, + data_timestamp, ) -> tuple: + where_sql, params = _acheteur_scope(pathname, ach_year) return prepare_table_data( - data, data_timestamp, filter_query, page_current, page_size, sort_by, "acheteur" + None, + data_timestamp, + filter_query, + page_current, + page_size, + sort_by, + "acheteur", + base_where_sql=where_sql, + base_params=params, ) @callback( Output(component_id="top10_titulaires", component_property="children"), - Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), ) -def get_top_titulaires(data): - table = get_top_org_table(data, "titulaire", ["titulaire_distance"]) +def get_top_titulaires(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + table = get_top_org_table( + query_marches(where_sql, params).lazy(), "titulaire", ["titulaire_distance"] + ) return make_card(fig=table, title="Top titulaires", lg=12, xl=12) @callback( Output("download-data-acheteur", "data"), Input("btn-download-data-acheteur", "n_clicks"), - State(component_id="acheteur_data", component_property="data"), - State(component_id="acheteur_nom", component_property="children"), + State(component_id="acheteur_url", component_property="pathname"), State(component_id="acheteur_year", component_property="value"), + State(component_id="acheteur_nom", component_property="children"), prevent_initial_call=True, ) def download_acheteur_data( n_clicks, - data: list[dict[str, Any]], - acheteur_nom: str, + pathname: str, annee: str, + acheteur_nom: str, ): - df_to_download = pl.DataFrame(data) + where_sql, params = _acheteur_scope(pathname, annee) + df_to_download = query_marches( + where_sql, params, order_by='"dateNotification" DESC, uid DESC' + ) def to_bytes(buffer): write_styled_excel( @@ -418,8 +445,9 @@ def download_acheteur_data( @callback( Output("acheteur-download-filtered-data", "data"), - State("acheteur_data", "data"), Input("btn-download-filtered-data-acheteur", "n_clicks"), + State("acheteur_url", "pathname"), + State("acheteur_year", "value"), State("acheteur_nom", "children"), State("acheteur_datatable", "filter_query"), State("acheteur_datatable", "sort_by"), @@ -427,16 +455,16 @@ def download_acheteur_data( prevent_initial_call=True, ) def download_filtered_acheteur_data( - data, n_clicks, + pathname, + ach_year, acheteur_nom, filter_query, sort_by, hidden_columns: list | None = None, ): - lff: pl.LazyFrame = pl.LazyFrame( - data - ) # start from the full acheteur data, not from paginated table data + where_sql, params = _acheteur_scope(pathname, ach_year) + lff: pl.LazyFrame = query_marches(where_sql, params).lazy() # Les colonnes masquées sont supprimées if hidden_columns: @@ -535,10 +563,12 @@ def reset_view(n_clicks): @callback( Output("acheteur-distance-histogram", "children"), - Input("acheteur_data", "data"), + Input("acheteur_url", "pathname"), + Input("acheteur_year", "value"), ) -def update_acheteur_distance_histogram(data): - lff = pl.LazyFrame(data, strict=False, infer_schema_length=5000) +def update_acheteur_distance_histogram(pathname, ach_year): + where_sql, params = _acheteur_scope(pathname, ach_year) + lff = query_marches(where_sql, params).lazy() fig = get_distance_histogram(lff) return make_card( title="Distance acheteur–titulaire", diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 6ce3eb1..d84ec4e 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -1,5 +1,4 @@ import datetime -from typing import Any import dash_bootstrap_components as dbc import polars as pl @@ -15,7 +14,7 @@ from dash import ( register_page, ) -from src.db import query_marches, schema +from src.db import aggregate_marches, count_marches, query_marches, schema from src.figures import ( DataTable, get_distance_histogram, @@ -47,6 +46,17 @@ def get_title(titulaire_id: str = None) -> str: return "Marchés publics remportés | colibre" +def _titulaire_scope(pathname: str, titulaire_year: str | None) -> tuple[str, list]: + """WHERE SQL scopant les requêtes à ce titulaire (et éventuellement une année).""" + titulaire_siret = pathname.split("/")[-1] + where_sql = "titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'" + params: list = [titulaire_siret] + if titulaire_year and titulaire_year != "Toutes les années": + where_sql += ' AND YEAR("dateNotification") = ?' + params.append(int(titulaire_year)) + return where_sql, params + + register_page( __name__, path_template="/titulaires/", @@ -74,7 +84,6 @@ DATATABLE = html.Div( ) layout = [ - dcc.Store(id="titulaire_data", storage_type="memory"), dcc.Store(id="titulaire-hidden-columns", storage_type="local"), dcc.Store(id="filter-cleanup-trigger-titulaire"), dcc.Location(id="titulaire_url", refresh="callback-nav"), @@ -167,6 +176,7 @@ layout = [ dbc.Col( id="titulaire_map", width=4, + style={"minHeight": "300px"}, ), ], ), @@ -179,12 +189,17 @@ layout = [ html.Div( className="marches_table", id="top10_acheteurs", + style={"minHeight": "420px"}, ), ], ), width=8, ), - dbc.Col(id="titulaire-distance-histogram", width=4), + dbc.Col( + id="titulaire-distance-histogram", + width=4, + style={"minHeight": "450px"}, + ), ], ), ], @@ -334,26 +349,25 @@ def update_titulaire_infos(url): Output( component_id="titulaire_acheteurs_differents", component_property="children" ), - Input(component_id="titulaire_data", component_property="data"), + Input(component_id="titulaire_url", component_property="pathname"), + Input(component_id="titulaire_year", component_property="value"), ) -def update_titulaire_stats(data): - dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) - if dff.height == 0: - nb_marches = 0 - nb_acheteurs = 0 - else: - df_marches = dff.unique("uid") - nb_marches = format_number(df_marches.height) - nb_acheteurs = dff.unique("acheteur_id").height +def update_titulaire_stats(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + agg = aggregate_marches( + "COUNT(DISTINCT uid) AS n, COUNT(DISTINCT acheteur_id) AS nb_acheteurs", + where_sql, + params, + ) + nb_marches = format_number(int(agg["n"][0])) if agg.height else "0" + nb_acheteurs = format_number(int(agg["nb_acheteurs"][0])) if agg.height else "0" texte_marches_remportes = [ html.Strong(nb_marches), " marchés et accord-cadres remportés", ] - # + ", pour un total de ", html.Strong(somme_marches + " €")] - texte_nb_acheteurs = [ - html.Strong(format_number(nb_acheteurs)), + html.Strong(nb_acheteurs), " acheteurs (SIRET) différents", ] @@ -361,29 +375,15 @@ def update_titulaire_stats(data): @callback( - Output(component_id="titulaire_data", component_property="data"), Output("btn-download-data-titulaire", "disabled"), Output("btn-download-data-titulaire", "children"), Output("btn-download-data-titulaire", "title"), Input(component_id="titulaire_url", component_property="pathname"), Input(component_id="titulaire_year", component_property="value"), ) -def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: - titulaire_siret = url.split("/")[-1] - lff = query_marches( - "titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'", - (titulaire_siret,), - ).lazy() - if titulaire_year and titulaire_year != "Toutes les années": - lff = lff.filter( - pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year) - ) - lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) - lff = lff.fill_null("") - dff: pl.DataFrame = lff.collect(engine="streaming") - download_disabled, download_text, download_title = get_button_properties(dff.height) - data = dff.to_dicts() - return data, download_disabled, download_text, download_title +def update_download_button_titulaire(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + return get_button_properties(count_marches(where_sql, params)) @callback( @@ -396,8 +396,8 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: Output("btn-download-filtered-data-titulaire", "children"), Output("btn-download-filtered-data-titulaire", "title"), Output("filter-cleanup-trigger-titulaire", "data"), - Input(component_id="titulaire_url", component_property="href"), - Input("titulaire_data", "data"), + Input("titulaire_url", "pathname"), + Input("titulaire_year", "value"), Input("titulaire_datatable", "page_current"), Input("titulaire_datatable", "page_size"), Input("titulaire_datatable", "filter_query"), @@ -405,42 +405,58 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: State("titulaire_datatable", "data_timestamp"), ) def get_last_marches_data( - href, data, page_current, page_size, filter_query, sort_by, data_timestamp + pathname, + titulaire_year, + page_current, + page_size, + filter_query, + sort_by, + data_timestamp, ) -> list[dict]: + where_sql, params = _titulaire_scope(pathname, titulaire_year) return prepare_table_data( - data, + None, data_timestamp, filter_query, page_current, page_size, sort_by, "titulaire", + base_where_sql=where_sql, + base_params=params, ) @callback( Output(component_id="top10_acheteurs", component_property="children"), - Input(component_id="titulaire_data", component_property="data"), + Input(component_id="titulaire_url", component_property="pathname"), + Input(component_id="titulaire_year", component_property="value"), ) -def get_top_acheteurs(data): - return get_top_org_table(data, "acheteur", ["titulaire_distance"]) +def get_top_acheteurs(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + return get_top_org_table( + query_marches(where_sql, params).lazy(), "acheteur", ["titulaire_distance"] + ) @callback( Output("download-data-titulaire", "data"), Input("btn-download-data-titulaire", "n_clicks"), - State(component_id="titulaire_data", component_property="data"), - State(component_id="titulaire_nom", component_property="children"), + State(component_id="titulaire_url", component_property="pathname"), State(component_id="titulaire_year", component_property="value"), + State(component_id="titulaire_nom", component_property="children"), prevent_initial_call=True, ) def download_titulaire_data( n_clicks, - data: list[dict[str, Any]], - titulaire_nom: str, + pathname: str, annee: str, + titulaire_nom: str, ): - df_to_download = pl.DataFrame(data) + where_sql, params = _titulaire_scope(pathname, annee) + df_to_download = query_marches( + where_sql, params, order_by='"dateNotification" DESC, uid DESC' + ).fill_null("") def to_bytes(buffer): write_styled_excel( @@ -455,8 +471,9 @@ def download_titulaire_data( @callback( Output("titulaire-download-filtered-data", "data"), - State("titulaire_data", "data"), Input("btn-download-filtered-data-titulaire", "n_clicks"), + State("titulaire_url", "pathname"), + State("titulaire_year", "value"), State("titulaire_nom", "children"), State("titulaire_datatable", "filter_query"), State("titulaire_datatable", "sort_by"), @@ -464,16 +481,16 @@ def download_titulaire_data( prevent_initial_call=True, ) def download_filtered_titulaire_data( - data, n_clicks, + pathname, + titulaire_year, titulaire_nom, filter_query, sort_by, hidden_columns: list | None = None, ): - lff: pl.LazyFrame = pl.LazyFrame( - data - ) # start from the full titulaire data, not from paginated table data + where_sql, params = _titulaire_scope(pathname, titulaire_year) + lff: pl.LazyFrame = query_marches(where_sql, params).lazy() # Les colonnes masquées sont supprimées if hidden_columns: @@ -572,10 +589,12 @@ def reset_view(n_clicks): @callback( Output("titulaire-distance-histogram", "children"), - Input("titulaire_data", "data"), + Input("titulaire_url", "pathname"), + Input("titulaire_year", "value"), ) -def update_titulaire_distance_histogram(data): - lff = pl.LazyFrame(data) +def update_titulaire_distance_histogram(pathname, titulaire_year): + where_sql, params = _titulaire_scope(pathname, titulaire_year) + lff = query_marches(where_sql, params).lazy() if "titulaire_distance" in lff.collect_schema().names(): lff = lff.with_columns( pl.col("titulaire_distance").cast(pl.Float64, strict=False) diff --git a/src/utils/table.py b/src/utils/table.py index 0298012..1d7681a 100644 --- a/src/utils/table.py +++ b/src/utils/table.py @@ -427,9 +427,14 @@ def _fetch_page_sql( sort_by_key: tuple, page_current: int, page_size: int, + base_where_sql: str = "TRUE", + base_params: tuple = (), ) -> tuple[pl.DataFrame, int, int]: """Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule. + `base_where_sql`/`base_params` permettent de scoper la requête (ex : un + acheteur ou un titulaire précis) en plus du filtre saisi dans la table. + Retourne (page_dataframe_post_traitée, total_count, total_unique_count). """ # Import local pour éviter une dépendance circulaire @@ -441,7 +446,9 @@ def _fetch_page_sql( f"page={page_current} size={page_size}" ) - where_sql, params = filter_query_to_sql(filter_query or "", schema) + filter_where_sql, filter_params = filter_query_to_sql(filter_query or "", schema) + where_sql = f"({base_where_sql}) AND ({filter_where_sql})" + params = [*base_params, *filter_params] sort_by_dash = [ {"column_id": col, "direction": direction} for col, direction in sort_by_key @@ -464,7 +471,15 @@ def _fetch_page_sql( def prepare_table_data( - data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table + data, + data_timestamp, + filter_query, + page_current, + page_size, + sort_by, + source_table, + base_where_sql: str = "TRUE", + base_params: tuple = (), ): """ Fonction de préparation des données pour les datatables, afin de permettre une gestion fine des logiques, @@ -476,6 +491,8 @@ def prepare_table_data( :param page_size: :param sort_by: :param source_table: + :param base_where_sql: scope SQL additionnel (ex : un acheteur/titulaire précis) + :param base_params: paramètres liés à base_where_sql :return: """ logger.debug(" + + + + + + + + + + + + + + + + + + ") @@ -486,13 +503,16 @@ def prepare_table_data( trigger_cleanup = no_update if source_table == "tableau" else str(uuid.uuid4()) if data is None: - # Probablement car il s'agit de la page Tableau + # Chemin rapide SQL : tableau, acheteur, titulaire (scope éventuel via + # base_where_sql/base_params) sort_by_key = normalize_sort_by(sort_by) dff, height, total_unique = _fetch_page_sql( filter_query=filter_query, sort_by_key=sort_by_key, page_current=page_current, page_size=page_size, + base_where_sql=base_where_sql, + base_params=tuple(base_params), ) else: if isinstance(data, list): diff --git a/tests/test_main.py b/tests/test_main.py index f88571f..b458dc3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -98,12 +98,10 @@ def test_003_tableau_download(dash_duo: DashComposite): # Juste pour instancier l'app print(app.server.name) - dicts = pl.read_parquet("tests/test.parquet").to_dicts() - outputs = [ download_data(1, "", [], None), - download_acheteur_data(1, dicts, "123", "2025"), - download_titulaire_data(1, dicts, "345", "2025"), + download_acheteur_data(1, "/acheteurs/123", "2025", "ACHETEUR 1"), + download_titulaire_data(1, "/titulaires/345", "2025", "TITULAIRE 1"), ] for output in outputs: assert isinstance(output, dict) @@ -118,8 +116,6 @@ def test_003_tableau_download(dash_duo: DashComposite): def test_004_add_links_observatoire_acheteur(): - import polars as pl - from src.utils.table import add_links dff = pl.DataFrame( @@ -143,8 +139,6 @@ def test_004_add_links_observatoire_acheteur(): def test_005_add_links_observatoire_titulaire(): - import polars as pl - from src.utils.table import add_links dff = pl.DataFrame( @@ -323,7 +317,6 @@ def test_011_observatoire_multi_param_url(dash_duo: DashComposite): def test_012_get_distance_histogram_returns_graph(): - import polars as pl from dash import dcc from src.figures import get_distance_histogram @@ -334,7 +327,6 @@ def test_012_get_distance_histogram_returns_graph(): def test_013_get_distance_histogram_handles_nulls(): - import polars as pl from dash import dcc from src.figures import get_distance_histogram @@ -345,7 +337,6 @@ def test_013_get_distance_histogram_handles_nulls(): def test_014_get_distance_histogram_all_nulls(): - import polars as pl from dash import dcc from src.figures import get_distance_histogram