From 53919bff3ec062758c621186cd2592cc05c35843 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 24 Nov 2025 10:41:47 +0100 Subject: [PATCH 1/6] Cast acheteur_year to int et non dateNotification to string --- src/pages/acheteur.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 0213f67..ba58bdc 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -197,9 +197,8 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: "dureeMois", ) if acheteur_year and acheteur_year != "Toutes": - lff = lff.filter( - pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) - ) + acheteur_year = int(acheteur_year) + lff = lff.filter(pl.col("dateNotification").dt.year() == acheteur_year) lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) data = lff.collect(engine="streaming").to_dicts() From 565a0cfd08a4402c69d50ead4ae5ef36245e496f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 24 Nov 2025 11:17:05 +0100 Subject: [PATCH 2/6] Custom filter logic --- src/pages/acheteur.py | 51 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index ba58bdc..520230d 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -8,6 +8,7 @@ from src.figures import DataTable, point_on_map from src.utils import ( add_links_in_dict, df, + filter_table_data, format_number, format_values, get_annuaire_data, @@ -26,6 +27,16 @@ register_page( order=5, ) +datatable = html.Div( + className="marches_table", + children=DataTable( + dtid="acheteur_datatable", + page_action="native", + filter_action="custom", + page_size=10, + ), +) + layout = [ dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), @@ -101,7 +112,7 @@ layout = [ ), # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics attribués"), - html.Div(id="acheteur_last_marches", children=""), + html.Div(id="acheteur_last_marches", children=datatable), ], ), ] @@ -200,43 +211,37 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: acheteur_year = int(acheteur_year) lff = lff.filter(pl.col("dateNotification").dt.year() == acheteur_year) lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) + lff = lff.fill_null("") data = lff.collect(engine="streaming").to_dicts() return data @callback( - Output(component_id="acheteur_last_marches", component_property="children"), + Output(component_id="acheteur_datatable", component_property="data"), + Output(component_id="acheteur_datatable", component_property="columns"), + Output(component_id="acheteur_datatable", component_property="tooltip_header"), Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_datatable", component_property="filter_query"), ) -def get_last_marches_table(data) -> html.Div: - dff = pl.DataFrame(data) - if dff.height == 0: - return html.Div(html.P("Aucun marché trouvé.")) - dff = dff.cast(pl.String) - dff = dff.fill_null("") - dff = format_values(dff) - columns, tooltip = setup_table_columns( +def get_last_marches_data(data, filter_query) -> tuple[list[dict], list, list]: + lff: pl.LazyFrame = pl.LazyFrame(data) + if filter_query: + lff = filter_table_data(lff, filter_query) + + lff = lff.cast(pl.String) + dff: pl.DataFrame = format_values(lff.collect()) + columns, tooltip_header = setup_table_columns( dff, hideable=False, exclude=["titulaire_id", "titulaire_typeIdentifiant", "uid"], ) + data = dff.to_dicts() + data = add_links_in_dict(data, "titulaire") - table = html.Div( - className="marches_table", - children=DataTable( - dtid="acheteur_datatable", - data=data, - page_action="native", - filter_action="native", - columns=columns, - tooltip_header=tooltip, - page_size=10, - ), - ) - return table + return data, columns, tooltip_header @callback( From 8c98a60c75463d6e37f0728a0b7fec38474f67f2 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 24 Nov 2025 13:43:42 +0100 Subject: [PATCH 3/6] filtres, tris fonctionnels (acheteur) --- src/assets/style.css | 8 ++- src/pages/acheteur.py | 89 +++++++++++++++------------- src/pages/tableau.py | 70 ++-------------------- src/utils.py | 134 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 180 insertions(+), 121 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index c009a16..4f91b10 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -133,6 +133,12 @@ td[data-dash-column="objet"],td[data-dash-column="titulaire_nom"],td[data-dash-c .marches_table { font-family: "Open Sans", sans-serif; } + +.marches_table.stuck { + position: relative; + right: 200px; +} + .marches_table .cell-table tr:nth-child(even) td { background-color: #feeeee; font-family: "Open Sans", sans-serif; @@ -212,7 +218,7 @@ summary > h3 { } #_pages_content { - padding-top: 28px; + padding: 28px 24px 0 24px; } /* Vue acheteur/titulaire/recherche */ diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 520230d..36c3295 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -6,15 +6,13 @@ from dash import Input, Output, State, callback, dcc, html, register_page from src.callbacks import get_top_org_table from src.figures import DataTable, point_on_map from src.utils import ( - add_links_in_dict, df, - filter_table_data, format_number, - format_values, get_annuaire_data, + get_default_hidden_columns, get_departement_region, meta_content, - setup_table_columns, + prepare_table_data, ) register_page( @@ -31,9 +29,11 @@ datatable = html.Div( className="marches_table", children=DataTable( dtid="acheteur_datatable", - page_action="native", + page_action="custom", filter_action="custom", + sort_action="custom", page_size=10, + hidden_columns=get_default_hidden_columns(page="acheteur"), ), ) @@ -41,7 +41,6 @@ layout = [ dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), html.Div( - className="container", children=[ html.Div( className="wrapper", @@ -112,7 +111,29 @@ layout = [ ), # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics attribués"), - html.Div(id="acheteur_last_marches", children=datatable), + dcc.Loading( + overlay_style={"visibility": "visible", "filter": "blur(2px)"}, + id="loading-home", + type="default", + children=[ + html.Div( + [ + html.P("lignes", id="acheteur_nb_rows"), + html.Button( + "Téléchargement désactivé au-delà de 65 000 lignes", + id="btn-download-data-acheteur", + disabled=True, + ), + dcc.Download(id="acheteur-download-data"), + dcc.Store( + id="acheteur_filtered_data", storage_type="memory" + ), + ], + className="table-menu", + ), + datatable, + ], + ), ], ), ] @@ -194,19 +215,6 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: acheteur_siret = url.split("/")[-1] lff = df.lazy() lff = lff.filter(pl.col("acheteur_id") == acheteur_siret) - lff = lff.select( - "id", - "uid", - "objet", - "dateNotification", - "titulaire_id", - "titulaire_typeIdentifiant", - "titulaire_nom", - "distance", - "montant", - "codeCPV", - "dureeMois", - ) if acheteur_year and acheteur_year != "Toutes": acheteur_year = int(acheteur_year) lff = lff.filter(pl.col("dateNotification").dt.year() == acheteur_year) @@ -218,31 +226,28 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: @callback( - Output(component_id="acheteur_datatable", component_property="data"), - Output(component_id="acheteur_datatable", component_property="columns"), - Output(component_id="acheteur_datatable", component_property="tooltip_header"), - Input(component_id="acheteur_data", component_property="data"), - Input(component_id="acheteur_datatable", component_property="filter_query"), + Output("acheteur_datatable", "data"), + Output("acheteur_datatable", "columns"), + Output("acheteur_datatable", "tooltip_header"), + Output("acheteur_datatable", "data_timestamp"), + Output("acheteur_nb_rows", "children"), + Output("btn-download-data-acheteur", "disabled"), + Output("btn-download-data-acheteur", "children"), + Output("btn-download-data-acheteur", "title"), + Input("acheteur_data", "data"), + Input("acheteur_datatable", "page_current"), + Input("acheteur_datatable", "page_size"), + Input("acheteur_datatable", "filter_query"), + Input("acheteur_datatable", "sort_by"), + State("acheteur_datatable", "data_timestamp"), ) -def get_last_marches_data(data, filter_query) -> tuple[list[dict], list, list]: - lff: pl.LazyFrame = pl.LazyFrame(data) - if filter_query: - lff = filter_table_data(lff, filter_query) - - lff = lff.cast(pl.String) - dff: pl.DataFrame = format_values(lff.collect()) - columns, tooltip_header = setup_table_columns( - dff, - hideable=False, - exclude=["titulaire_id", "titulaire_typeIdentifiant", "uid"], +def get_last_marches_data( + data, page_current, page_size, filter_query, sort_by, data_timestamp +) -> list[dict]: + return prepare_table_data( + data, data_timestamp, filter_query, page_current, page_size, sort_by ) - data = dff.to_dicts() - - data = add_links_in_dict(data, "titulaire") - - return data, columns, tooltip_header - @callback( Output(component_id="top10_titulaires", component_property="children"), diff --git a/src/pages/tableau.py b/src/pages/tableau.py index ab3f712..ba938b5 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -6,22 +6,17 @@ from dash import Input, Output, State, callback, dcc, html, register_page from src.figures import DataTable from src.utils import ( - add_links, - add_resource_link, df, filter_table_data, - format_number, - format_values, get_default_hidden_columns, meta_content, - setup_table_columns, sort_table_data, ) +from utils import prepare_table_data update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -schema = df.collect_schema() name = "Tableau" register_page( @@ -42,7 +37,7 @@ datatable = html.Div( page_action="custom", filter_action="custom", sort_action="custom", - hidden_columns=get_default_hidden_columns(schema), + hidden_columns=get_default_hidden_columns(None), ), ) @@ -138,65 +133,8 @@ layout = [ State("table", "data_timestamp"), ) def update_table(page_current, page_size, filter_query, sort_by, data_timestamp): - if os.getenv("DEVELOPMENT").lower() == "true": - print(" + + + + + + + + + + + + + + + + + + ") - - # Application des filtres - lff: pl.LazyFrame = df.lazy() # start from the original data - if filter_query: - lff = filter_table_data(lff, filter_query) - - if len(sort_by) > 0: - lff = sort_table_data(lff, sort_by) - - # Matérialisation des filtres - dff: pl.DataFrame = lff.collect() - - height = dff.height - nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)" - - # Pagination des données - start_row = page_current * page_size - # end_row = (page_current + 1) * page_size - dff = dff.slice(start_row, page_size) - - # Tout devient string - dff = dff.cast(pl.String) - - # Remplace les strings null par "", mais pas les numeric null - dff = dff.fill_null("") - - # Ajout des liens vers l'annuaire des entreprises - dff = add_links(dff) - - # Ajout des liens vers les fichiers Open Data - dff = add_resource_link(dff) - - # Formatage des montants - dff = format_values(dff) - - columns, tooltip = setup_table_columns(dff) - - dicts = dff.to_dicts() - - if height > 65000: - download_disabled = True - download_text = "Téléchargement désactivé au-delà de 65 000 lignes" - download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." - else: - download_disabled = False - download_text = "Télécharger au format Excel" - download_title = "" - - return ( - dicts, - columns, - tooltip, - data_timestamp + 1, - nb_rows, - download_disabled, - download_text, - download_title, + return prepare_table_data( + None, data_timestamp, filter_query, page_current, page_size, sort_by ) diff --git a/src/utils.py b/src/utils.py index 084938c..d520f74 100644 --- a/src/utils.py +++ b/src/utils.py @@ -7,7 +7,6 @@ from time import localtime, sleep import polars as pl import polars.selectors as cs from httpx import get, post -from polars import Schema from polars.exceptions import ComputeError from unidecode import unidecode @@ -356,18 +355,47 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup return columns, tooltip -def get_default_hidden_columns(schema: Schema): - displayed_columns = os.getenv("DISPLAYED_COLUMNS") +def get_default_hidden_columns(page): + if page == "acheteur": + displayed_columns = [ + "uid", + "objet", + "dateNotification", + "titulaire_id", + "titulaire_typeIdentifiant", + "titulaire_nom", + "distance", + "montant", + "codeCPV", + "dureeRestanteMois", + ] + elif page == "titulaire": + displayed_columns = [ + "uid", + "objet", + "dateNotification", + "acheteur_id", + "acheteur_nom", + "distance", + "montant", + "codeCPV", + "dureeRestanteMois", + ] + else: + displayed_columns = os.getenv("DISPLAYED_COLUMNS") + if displayed_columns is None: + raise ValueError("DISPLAYED_COLUMNS n'est pas configuré") + else: + displayed_columns = displayed_columns.replace(" ", "").split(",") + hidden_columns = [] - if displayed_columns: - displayed_columns = displayed_columns.replace(" ", "").split(",") - for col in schema.names(): - if col in displayed_columns: - continue - else: - hidden_columns.append(col) - return hidden_columns - raise ValueError("DISPLAYED_COLUMNS n'est pas configuré") + + for col in schema.names(): + if col in displayed_columns: + continue + else: + hidden_columns.append(col) + return hidden_columns def get_data_schema() -> dict: @@ -503,9 +531,91 @@ def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame: return dff +def prepare_table_data( + data, data_timestamp, filter_query, page_current, page_size, sort_by +): + """ + Fonction de préparation des données pour les datatables, afin de permettre une gestion fine des logiques, + notamment pour les filtres et les tris. + :param data + :param data_timestamp: + :param filter_query: + :param page_current: + :param page_size: + :param sort_by: + :return: + """ + + if os.getenv("DEVELOPMENT").lower() == "true": + print(" + + + + + + + + + + + + + + + + + + ") + + # Récupération des données + if data: + lff: pl.LazyFrame = pl.LazyFrame(data) + else: + lff: pl.LazyFrame = df.lazy() # start from the original data + + # Application des filtres + if filter_query: + lff = filter_table_data(lff, filter_query) + + # Application des tris + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) + + # Matérialisation des filtres + dff: pl.DataFrame = lff.collect() + height = dff.height + nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)" + + # Pagination des données + start_row = page_current * page_size + # end_row = (page_current + 1) * page_size + dff = dff.slice(start_row, page_size) + + # Tout devient string + dff = dff.cast(pl.String) + + # Remplace les strings null par "", mais pas les numeric null + dff = dff.fill_null("") + + # Ajout des liens vers l'annuaire des entreprises + dff = add_links(dff) + + # Ajout des liens vers les fichiers Open Data + if "sourceFile" in dff.columns: + dff = add_resource_link(dff) + + # Formatage des montants + dff = format_values(dff) + columns, tooltip = setup_table_columns(dff) + dicts = dff.to_dicts() + if height > 65000: + download_disabled = True + download_text = "Téléchargement désactivé au-delà de 65 000 lignes" + download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." + else: + download_disabled = False + download_text = "Télécharger au format Excel" + download_title = "" + return ( + dicts, + columns, + tooltip, + data_timestamp + 1, + nb_rows, + download_disabled, + download_text, + download_title, + ) + + df: pl.DataFrame = get_decp_data() +schema = df.collect_schema() + df_acheteurs = get_org_data(df, "acheteur") df_titulaires = get_org_data(df, "titulaire") + departements = get_departements() domain_name = ( "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" From ec1585240601f34a1b0e5e7697536abed87c27a8 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 24 Nov 2025 14:00:28 +0100 Subject: [PATCH 4/6] =?UTF-8?q?T=C3=A9l=C3=A9chargemetn=20filtr=C3=A9=20OK?= =?UTF-8?q?=20dans=20acheteur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 36c3295..db9245e 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -7,12 +7,14 @@ from src.callbacks import get_top_org_table from src.figures import DataTable, point_on_map from src.utils import ( df, + filter_table_data, format_number, get_annuaire_data, get_default_hidden_columns, get_departement_region, meta_content, prepare_table_data, + sort_table_data, ) register_page( @@ -97,6 +99,7 @@ layout = [ id="btn-download-acheteur-data", ), dcc.Download(id="download-acheteur-data"), + dcc.Download(id="download-acheteur-data-filtered"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -280,3 +283,39 @@ def download_acheteur_data( date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return dcc.send_bytes(to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx") + + +@callback( + Output("download-acheteur-data-filtered", "data"), + State("acheteur_data", "data"), + Input("btn-download-data-acheteur", "n_clicks"), + State("acheteur_nom", "children"), + State("acheteur_datatable", "filter_query"), + State("acheteur_datatable", "sort_by"), + State("acheteur_datatable", "hidden_columns"), + prevent_initial_call=True, +) +def download_filtered_acheteur_data( + data, n_clicks, acheteur_nom, filter_query, sort_by, hidden_columns: list = None +): + lff: pl.LazyFrame = pl.LazyFrame( + data + ) # start from the full acheteur data, not from paginated table data + + # Les colonnes masquées sont supprimées + if hidden_columns: + lff = lff.drop(hidden_columns) + + if filter_query: + lff = filter_table_data(lff, filter_query) + + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) + + def to_bytes(buffer): + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes( + to_bytes, filename=f"decp_filtrées_{acheteur_nom}_{date}.xlsx" + ) From 7f42ca67a22de8e0399cf302ceff8cc9d183e686 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 24 Nov 2025 15:33:27 +0100 Subject: [PATCH 5/6] =?UTF-8?q?Bonne=20configuration=20des=20boutons=20de?= =?UTF-8?q?=20t=C3=A9l=C3=A9chargemetn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +- pyproject.toml | 4 +- src/pages/acheteur.py | 41 +++++------ src/pages/titulaire.py | 157 +++++++++++++++++++++++++++-------------- src/utils.py | 31 +++++--- 5 files changed, 152 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index 1d03d66..f31edfa 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # decp.info -> v2.2.1 - -Outil d'exploration et de téléchargement des données essentielles de la commande publique. +> v2.3.0 +> Outil d'exploration et de téléchargement des données essentielles de la commande publique. => [decp.info](https://decp.info) diff --git a/pyproject.toml b/pyproject.toml index d0c093f..5cd1e4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "decp.info" description = "Interface d'exploration et d'analyse des marchés publics français." -version = "2.2.1" +version = "2.3.0" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin+decp@maudry.com" } ] dependencies = [ - "dash==3.2.0", + "dash==3.3.0", "dash[compress]", "polars", "gunicorn", diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index db9245e..8d2eca0 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -10,6 +10,7 @@ from src.utils import ( filter_table_data, format_number, get_annuaire_data, + get_button_properties, get_default_hidden_columns, get_departement_region, meta_content, @@ -96,10 +97,9 @@ layout = [ html.P(id="acheteur_titulaires_differents"), html.Button( "Téléchargement au format Excel", - id="btn-download-acheteur-data", + id="btn-download-data-acheteur", ), - dcc.Download(id="download-acheteur-data"), - dcc.Download(id="download-acheteur-data-filtered"), + dcc.Download(id="download-data-acheteur"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -124,13 +124,10 @@ layout = [ html.P("lignes", id="acheteur_nb_rows"), html.Button( "Téléchargement désactivé au-delà de 65 000 lignes", - id="btn-download-data-acheteur", + id="btn-download-filtered-data-acheteur", disabled=True, ), - dcc.Download(id="acheteur-download-data"), - dcc.Store( - id="acheteur_filtered_data", storage_type="memory" - ), + dcc.Download(id="acheteur-download-filtered-data"), ], className="table-menu", ), @@ -211,21 +208,25 @@ def update_acheteur_stats(data): @callback( Output(component_id="acheteur_data", component_property="data"), + Output("btn-download-data-acheteur", "disabled"), + Output("btn-download-data-acheteur", "children"), + Output("btn-download-data-acheteur", "title"), Input(component_id="url", component_property="pathname"), Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: +def get_acheteur_marches_data(url, acheteur_year: str) -> tuple: acheteur_siret = url.split("/")[-1] lff = df.lazy() lff = lff.filter(pl.col("acheteur_id") == acheteur_siret) if acheteur_year and acheteur_year != "Toutes": acheteur_year = int(acheteur_year) lff = lff.filter(pl.col("dateNotification").dt.year() == acheteur_year) - lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) - lff = lff.fill_null("") + lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) + dff: pl.DataFrame = lff.collect(engine="streaming") + download_disabled, download_text, download_title = get_button_properties(dff.height) - data = lff.collect(engine="streaming").to_dicts() - return data + data = dff.to_dicts() + return data, download_disabled, download_text, download_title @callback( @@ -234,9 +235,9 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: Output("acheteur_datatable", "tooltip_header"), Output("acheteur_datatable", "data_timestamp"), Output("acheteur_nb_rows", "children"), - Output("btn-download-data-acheteur", "disabled"), - Output("btn-download-data-acheteur", "children"), - Output("btn-download-data-acheteur", "title"), + Output("btn-download-filtered-data-acheteur", "disabled"), + Output("btn-download-filtered-data-acheteur", "children"), + Output("btn-download-filtered-data-acheteur", "title"), Input("acheteur_data", "data"), Input("acheteur_datatable", "page_current"), Input("acheteur_datatable", "page_size"), @@ -261,8 +262,8 @@ def get_top_titulaires(data): @callback( - Output("download-acheteur-data", "data"), - Input("btn-download-acheteur-data", "n_clicks"), + Output("download-data-acheteur", "data"), + Input("btn-download-data-acheteur", "n_clicks"), State(component_id="acheteur_data", component_property="data"), State(component_id="acheteur_nom", component_property="children"), State(component_id="acheteur_year", component_property="value"), @@ -286,9 +287,9 @@ def download_acheteur_data( @callback( - Output("download-acheteur-data-filtered", "data"), + Output("acheteur-download-filtered-data", "data"), State("acheteur_data", "data"), - Input("btn-download-data-acheteur", "n_clicks"), + Input("btn-download-filtered-data-acheteur", "n_clicks"), State("acheteur_nom", "children"), State("acheteur_datatable", "filter_query"), State("acheteur_datatable", "sort_by"), diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 95f8c8f..ae1cca3 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -6,14 +6,16 @@ from dash import Input, Output, State, callback, dcc, html, register_page from src.callbacks import get_top_org_table from src.figures import DataTable, point_on_map from src.utils import ( - add_links_in_dict, df, + filter_table_data, format_number, - format_values, get_annuaire_data, + get_button_properties, + get_default_hidden_columns, get_departement_region, meta_content, - setup_table_columns, + prepare_table_data, + sort_table_data, ) register_page( @@ -26,13 +28,22 @@ register_page( order=5, ) -# 21690123100011 +datatable = html.Div( + className="marches_table", + children=DataTable( + dtid="titulaire_datatable", + page_action="custom", + filter_action="custom", + sort_action="custom", + page_size=10, + hidden_columns=get_default_hidden_columns(page="titulaire"), + ), +) layout = [ dcc.Store(id="titulaire_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), html.Div( - className="container", children=[ html.Div( className="wrapper", @@ -86,9 +97,9 @@ layout = [ html.P(id="titulaire_acheteurs_differents"), html.Button( "Téléchargement au format Excel", - id="btn-download-titulaire-data", + id="btn-download-data-titulaire", ), - dcc.Download(id="download-titulaire-data"), + dcc.Download(id="download-data-titulaire"), ], ), html.Div(className="org_map", id="titulaire_map"), @@ -103,7 +114,26 @@ layout = [ ), # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics remportés"), - html.Div(id="titulaire_last_marches", children=""), + dcc.Loading( + overlay_style={"visibility": "visible", "filter": "blur(2px)"}, + id="loading-home", + type="default", + children=[ + html.Div( + [ + html.P("lignes", id="titulaire_nb_rows"), + html.Button( + "Téléchargement désactivé au-delà de 65 000 lignes", + id="btn-download-filtered-data-titulaire", + disabled=True, + ), + dcc.Download(id="titulaire-download-filtered-data"), + ], + className="table-menu", + ), + datatable, + ], + ), ], ), ] @@ -178,68 +208,55 @@ def update_titulaire_stats(data): @callback( Output(component_id="titulaire_data", component_property="data"), + Output("btn-download-data-titulaire", "disabled"), + Output("btn-download-data-titulaire", "children"), + Output("btn-download-data-titulaire", "title"), Input(component_id="url", component_property="pathname"), Input(component_id="titulaire_year", component_property="value"), ) -def get_titulaire_marches_data(url, titulaire_year: str) -> list[dict]: +def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: titulaire_siret = url.split("/")[-1] lff = df.lazy() lff = lff.filter( (pl.col("titulaire_id") == titulaire_siret) & (pl.col("titulaire_typeIdentifiant") == "SIRET") ) - lff = lff.select( - "id", - "uid", - "objet", - "dateNotification", - "acheteur_id", - "acheteur_nom", - "distance", - "montant", - "codeCPV", - "dureeMois", - ) if titulaire_year and titulaire_year != "Toutes": lff = lff.filter( pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year) ) lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) + lff = lff.fill_null("") - data = lff.collect(engine="streaming").to_dicts() - return data + dff: pl.DataFrame = lff.collect(engine="streaming") + download_disabled, download_text, download_title = get_button_properties(dff.height) + + data = dff.to_dicts() + return data, download_disabled, download_text, download_title @callback( - Output(component_id="titulaire_last_marches", component_property="children"), - Input(component_id="titulaire_data", component_property="data"), + Output("titulaire_datatable", "data"), + Output("titulaire_datatable", "columns"), + Output("titulaire_datatable", "tooltip_header"), + Output("titulaire_datatable", "data_timestamp"), + Output("titulaire_nb_rows", "children"), + Output("btn-download-filtered-data-titulaire", "disabled"), + Output("btn-download-filtered-data-titulaire", "children"), + Output("btn-download-filtered-data-titulaire", "title"), + Input("titulaire_data", "data"), + Input("titulaire_datatable", "page_current"), + Input("titulaire_datatable", "page_size"), + Input("titulaire_datatable", "filter_query"), + Input("titulaire_datatable", "sort_by"), + State("titulaire_datatable", "data_timestamp"), ) -def get_last_marches_table(data) -> html.Div: - dff = pl.DataFrame(data) - dff = dff.cast(pl.String) - dff = dff.fill_null("") - dff = format_values(dff) - columns, tooltip = setup_table_columns( - dff, hideable=False, exclude=["acheteur_id", "id"] +def get_last_marches_data( + data, page_current, page_size, filter_query, sort_by, data_timestamp +) -> list[dict]: + return prepare_table_data( + data, data_timestamp, filter_query, page_current, page_size, sort_by ) - data = dff.to_dicts() - # Idéalement on utiliserait add_org_links(), mais le résultat attendu - # est différent de home.py (Tableau) - data = add_links_in_dict(data, "acheteur") - - table = html.Div( - className="marches_table", - children=DataTable( - dtid="titulaire_data_table", - data=data, - page_action="native", - filter_action="native", - columns=columns, - tooltip_header=tooltip, - page_size=10, - ), - ) - return table @callback( @@ -251,8 +268,8 @@ def get_top_acheteurs(data): @callback( - Output("download-titulaire-data", "data"), - Input("btn-download-titulaire-data", "n_clicks"), + Output("download-data-titulaire", "data"), + Input("btn-download-data-titulaire", "n_clicks"), State(component_id="titulaire_data", component_property="data"), State(component_id="titulaire_nom", component_property="children"), State(component_id="titulaire_year", component_property="value"), @@ -273,3 +290,39 @@ def download_titulaire_data( date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return dcc.send_bytes(to_bytes, filename=f"decp_{titulaire_nom}_{date}.xlsx") + + +@callback( + Output("titulaire-download-filtered-data", "data"), + State("titulaire_data", "data"), + Input("btn-download-filtered-data-titulaire", "n_clicks"), + State("titulaire_nom", "children"), + State("titulaire_datatable", "filter_query"), + State("titulaire_datatable", "sort_by"), + State("titulaire_datatable", "hidden_columns"), + prevent_initial_call=True, +) +def download_filtered_titulaire_data( + data, n_clicks, titulaire_nom, filter_query, sort_by, hidden_columns: list = None +): + lff: pl.LazyFrame = pl.LazyFrame( + data + ) # start from the full titulaire data, not from paginated table data + + # Les colonnes masquées sont supprimées + if hidden_columns: + lff = lff.drop(hidden_columns) + + if filter_query: + lff = filter_table_data(lff, filter_query) + + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) + + def to_bytes(buffer): + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes( + to_bytes, filename=f"decp_filtrées_{titulaire_nom}_{date}.xlsx" + ) diff --git a/src/utils.py b/src/utils.py index d520f74..fecbd76 100644 --- a/src/utils.py +++ b/src/utils.py @@ -330,9 +330,10 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup continue column_object = data_schema.get(column_id) if column_object: - column_name = column_object.get("title", column_id) + column_name = column_object.get("title") else: column_name = column_id + print("Colonne inconnue dans le schéma !", column_id) column = { "name": column_name, @@ -588,16 +589,15 @@ def prepare_table_data( # Formatage des montants dff = format_values(dff) + + # Récupération des colonnes et tooltip columns, tooltip = setup_table_columns(dff) + dicts = dff.to_dicts() - if height > 65000: - download_disabled = True - download_text = "Téléchargement désactivé au-delà de 65 000 lignes" - download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." - else: - download_disabled = False - download_text = "Télécharger au format Excel" - download_title = "" + + # Propriétés du bouton de téléchargement + download_disabled, download_text, download_title = get_button_properties(height) + return ( dicts, columns, @@ -610,6 +610,19 @@ def prepare_table_data( ) +def get_button_properties(height): + if height > 65000: + download_disabled = True + download_text = "Téléchargement désactivé au-delà de 65 000 lignes" + download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." + else: + print("moins de 65k") + download_disabled = False + download_text = "Télécharger au format Excel" + download_title = "" + return download_disabled, download_text, download_title + + df: pl.DataFrame = get_decp_data() schema = df.collect_schema() From e216b42379fc96fbf4cffa96921ecb9a4031953f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 24 Nov 2025 16:00:21 +0100 Subject: [PATCH 6/6] Meilleure gestion des df vides acheteur/titulaire --- src/pages/acheteur.py | 2 +- src/pages/titulaire.py | 25 ++++++++++++++----------- src/utils.py | 17 ++++++++++++----- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 8d2eca0..24b71cb 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -247,7 +247,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> tuple: ) def get_last_marches_data( data, page_current, page_size, filter_query, sort_by, data_timestamp -) -> list[dict]: +) -> tuple: return prepare_table_data( data, data_timestamp, filter_query, page_current, page_size, sort_by ) diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index ae1cca3..433fbe8 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -188,22 +188,25 @@ def update_titulaire_infos(url): def update_titulaire_stats(data): dff = pl.DataFrame(data) if dff.height == 0: - dff = pl.DataFrame(schema=dff.collect_schema()) - df_marches = dff.unique("uid") - nb_marches = format_number(df_marches.height) - # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) - marches_remportes = [html.Strong(nb_marches), " marchés et accord-cadres remportés"] - # + ", pour un total de ", html.Strong(somme_marches + " €")] - del df_marches + nb_marches = 0 + nb_acheteurs = 0 + else: + df_marches = dff.unique("uid") + nb_marches = format_number(df_marches.height) + nb_acheteurs = dff.unique("acheteur_id").height - nb_acheteurs = dff.unique("acheteur_id").height - nb_acheteurs = [ + texte_marches_remportes = [ + html.Strong(nb_marches), + " marchés et accord-cadres remportés", + ] + # + ", pour un total de ", html.Strong(somme_marches + " €")] + + texte_nb_acheteurs = [ html.Strong(format_number(nb_acheteurs)), " acheteurs (SIRET) différents", ] - del dff - return marches_remportes, nb_acheteurs + return texte_marches_remportes, texte_nb_acheteurs @callback( diff --git a/src/utils.py b/src/utils.py index fecbd76..19ab81c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -333,7 +333,6 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup column_name = column_object.get("title") else: column_name = column_id - print("Colonne inconnue dans le schéma !", column_id) column = { "name": column_name, @@ -551,7 +550,7 @@ def prepare_table_data( print(" + + + + + + + + + + + + + + + + + + ") # Récupération des données - if data: + if isinstance(data, list): lff: pl.LazyFrame = pl.LazyFrame(data) else: lff: pl.LazyFrame = df.lazy() # start from the original data @@ -567,7 +566,11 @@ def prepare_table_data( # Matérialisation des filtres dff: pl.DataFrame = lff.collect() height = dff.height - nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)" + + if height > 0: + nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)" + else: + nb_rows = "0 lignes (0 marchés)" # Pagination des données start_row = page_current * page_size @@ -588,7 +591,8 @@ def prepare_table_data( dff = add_resource_link(dff) # Formatage des montants - dff = format_values(dff) + if height > 0: + dff = format_values(dff) # Récupération des colonnes et tooltip columns, tooltip = setup_table_columns(dff) @@ -615,8 +619,11 @@ def get_button_properties(height): download_disabled = True download_text = "Téléchargement désactivé au-delà de 65 000 lignes" download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." + elif height == 0: + download_disabled = True + download_text = "Pas de données à télécharger" + download_title = "" else: - print("moins de 65k") download_disabled = False download_text = "Télécharger au format Excel" download_title = ""