diff --git a/README.md b/README.md index 1d03d66..f31edfa 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # decp.info -> v2.2.1 - -Outil d'exploration et de téléchargement des données essentielles de la commande publique. +> v2.3.0 +> Outil d'exploration et de téléchargement des données essentielles de la commande publique. => [decp.info](https://decp.info) diff --git a/pyproject.toml b/pyproject.toml index d0c093f..5cd1e4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "decp.info" description = "Interface d'exploration et d'analyse des marchés publics français." -version = "2.2.1" +version = "2.3.0" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin+decp@maudry.com" } ] dependencies = [ - "dash==3.2.0", + "dash==3.3.0", "dash[compress]", "polars", "gunicorn", diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index db9245e..8d2eca0 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -10,6 +10,7 @@ from src.utils import ( filter_table_data, format_number, get_annuaire_data, + get_button_properties, get_default_hidden_columns, get_departement_region, meta_content, @@ -96,10 +97,9 @@ layout = [ html.P(id="acheteur_titulaires_differents"), html.Button( "Téléchargement au format Excel", - id="btn-download-acheteur-data", + id="btn-download-data-acheteur", ), - dcc.Download(id="download-acheteur-data"), - dcc.Download(id="download-acheteur-data-filtered"), + dcc.Download(id="download-data-acheteur"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -124,13 +124,10 @@ layout = [ html.P("lignes", id="acheteur_nb_rows"), html.Button( "Téléchargement désactivé au-delà de 65 000 lignes", - id="btn-download-data-acheteur", + id="btn-download-filtered-data-acheteur", disabled=True, ), - dcc.Download(id="acheteur-download-data"), - dcc.Store( - id="acheteur_filtered_data", storage_type="memory" - ), + dcc.Download(id="acheteur-download-filtered-data"), ], className="table-menu", ), @@ -211,21 +208,25 @@ def update_acheteur_stats(data): @callback( Output(component_id="acheteur_data", component_property="data"), + Output("btn-download-data-acheteur", "disabled"), + Output("btn-download-data-acheteur", "children"), + Output("btn-download-data-acheteur", "title"), Input(component_id="url", component_property="pathname"), Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: +def get_acheteur_marches_data(url, acheteur_year: str) -> tuple: acheteur_siret = url.split("/")[-1] lff = df.lazy() lff = lff.filter(pl.col("acheteur_id") == acheteur_siret) if acheteur_year and acheteur_year != "Toutes": acheteur_year = int(acheteur_year) lff = lff.filter(pl.col("dateNotification").dt.year() == acheteur_year) - lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) - lff = lff.fill_null("") + lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) + dff: pl.DataFrame = lff.collect(engine="streaming") + download_disabled, download_text, download_title = get_button_properties(dff.height) - data = lff.collect(engine="streaming").to_dicts() - return data + data = dff.to_dicts() + return data, download_disabled, download_text, download_title @callback( @@ -234,9 +235,9 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: Output("acheteur_datatable", "tooltip_header"), Output("acheteur_datatable", "data_timestamp"), Output("acheteur_nb_rows", "children"), - Output("btn-download-data-acheteur", "disabled"), - Output("btn-download-data-acheteur", "children"), - Output("btn-download-data-acheteur", "title"), + Output("btn-download-filtered-data-acheteur", "disabled"), + Output("btn-download-filtered-data-acheteur", "children"), + Output("btn-download-filtered-data-acheteur", "title"), Input("acheteur_data", "data"), Input("acheteur_datatable", "page_current"), Input("acheteur_datatable", "page_size"), @@ -261,8 +262,8 @@ def get_top_titulaires(data): @callback( - Output("download-acheteur-data", "data"), - Input("btn-download-acheteur-data", "n_clicks"), + Output("download-data-acheteur", "data"), + Input("btn-download-data-acheteur", "n_clicks"), State(component_id="acheteur_data", component_property="data"), State(component_id="acheteur_nom", component_property="children"), State(component_id="acheteur_year", component_property="value"), @@ -286,9 +287,9 @@ def download_acheteur_data( @callback( - Output("download-acheteur-data-filtered", "data"), + Output("acheteur-download-filtered-data", "data"), State("acheteur_data", "data"), - Input("btn-download-data-acheteur", "n_clicks"), + Input("btn-download-filtered-data-acheteur", "n_clicks"), State("acheteur_nom", "children"), State("acheteur_datatable", "filter_query"), State("acheteur_datatable", "sort_by"), diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 95f8c8f..ae1cca3 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -6,14 +6,16 @@ from dash import Input, Output, State, callback, dcc, html, register_page from src.callbacks import get_top_org_table from src.figures import DataTable, point_on_map from src.utils import ( - add_links_in_dict, df, + filter_table_data, format_number, - format_values, get_annuaire_data, + get_button_properties, + get_default_hidden_columns, get_departement_region, meta_content, - setup_table_columns, + prepare_table_data, + sort_table_data, ) register_page( @@ -26,13 +28,22 @@ register_page( order=5, ) -# 21690123100011 +datatable = html.Div( + className="marches_table", + children=DataTable( + dtid="titulaire_datatable", + page_action="custom", + filter_action="custom", + sort_action="custom", + page_size=10, + hidden_columns=get_default_hidden_columns(page="titulaire"), + ), +) layout = [ dcc.Store(id="titulaire_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), html.Div( - className="container", children=[ html.Div( className="wrapper", @@ -86,9 +97,9 @@ layout = [ html.P(id="titulaire_acheteurs_differents"), html.Button( "Téléchargement au format Excel", - id="btn-download-titulaire-data", + id="btn-download-data-titulaire", ), - dcc.Download(id="download-titulaire-data"), + dcc.Download(id="download-data-titulaire"), ], ), html.Div(className="org_map", id="titulaire_map"), @@ -103,7 +114,26 @@ layout = [ ), # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics remportés"), - html.Div(id="titulaire_last_marches", children=""), + dcc.Loading( + overlay_style={"visibility": "visible", "filter": "blur(2px)"}, + id="loading-home", + type="default", + children=[ + html.Div( + [ + html.P("lignes", id="titulaire_nb_rows"), + html.Button( + "Téléchargement désactivé au-delà de 65 000 lignes", + id="btn-download-filtered-data-titulaire", + disabled=True, + ), + dcc.Download(id="titulaire-download-filtered-data"), + ], + className="table-menu", + ), + datatable, + ], + ), ], ), ] @@ -178,68 +208,55 @@ def update_titulaire_stats(data): @callback( Output(component_id="titulaire_data", component_property="data"), + Output("btn-download-data-titulaire", "disabled"), + Output("btn-download-data-titulaire", "children"), + Output("btn-download-data-titulaire", "title"), Input(component_id="url", component_property="pathname"), Input(component_id="titulaire_year", component_property="value"), ) -def get_titulaire_marches_data(url, titulaire_year: str) -> list[dict]: +def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: titulaire_siret = url.split("/")[-1] lff = df.lazy() lff = lff.filter( (pl.col("titulaire_id") == titulaire_siret) & (pl.col("titulaire_typeIdentifiant") == "SIRET") ) - lff = lff.select( - "id", - "uid", - "objet", - "dateNotification", - "acheteur_id", - "acheteur_nom", - "distance", - "montant", - "codeCPV", - "dureeMois", - ) if titulaire_year and titulaire_year != "Toutes": lff = lff.filter( pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year) ) lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) + lff = lff.fill_null("") - data = lff.collect(engine="streaming").to_dicts() - return data + dff: pl.DataFrame = lff.collect(engine="streaming") + download_disabled, download_text, download_title = get_button_properties(dff.height) + + data = dff.to_dicts() + return data, download_disabled, download_text, download_title @callback( - Output(component_id="titulaire_last_marches", component_property="children"), - Input(component_id="titulaire_data", component_property="data"), + Output("titulaire_datatable", "data"), + Output("titulaire_datatable", "columns"), + Output("titulaire_datatable", "tooltip_header"), + Output("titulaire_datatable", "data_timestamp"), + Output("titulaire_nb_rows", "children"), + Output("btn-download-filtered-data-titulaire", "disabled"), + Output("btn-download-filtered-data-titulaire", "children"), + Output("btn-download-filtered-data-titulaire", "title"), + Input("titulaire_data", "data"), + Input("titulaire_datatable", "page_current"), + Input("titulaire_datatable", "page_size"), + Input("titulaire_datatable", "filter_query"), + Input("titulaire_datatable", "sort_by"), + State("titulaire_datatable", "data_timestamp"), ) -def get_last_marches_table(data) -> html.Div: - dff = pl.DataFrame(data) - dff = dff.cast(pl.String) - dff = dff.fill_null("") - dff = format_values(dff) - columns, tooltip = setup_table_columns( - dff, hideable=False, exclude=["acheteur_id", "id"] +def get_last_marches_data( + data, page_current, page_size, filter_query, sort_by, data_timestamp +) -> list[dict]: + return prepare_table_data( + data, data_timestamp, filter_query, page_current, page_size, sort_by ) - data = dff.to_dicts() - # Idéalement on utiliserait add_org_links(), mais le résultat attendu - # est différent de home.py (Tableau) - data = add_links_in_dict(data, "acheteur") - - table = html.Div( - className="marches_table", - children=DataTable( - dtid="titulaire_data_table", - data=data, - page_action="native", - filter_action="native", - columns=columns, - tooltip_header=tooltip, - page_size=10, - ), - ) - return table @callback( @@ -251,8 +268,8 @@ def get_top_acheteurs(data): @callback( - Output("download-titulaire-data", "data"), - Input("btn-download-titulaire-data", "n_clicks"), + Output("download-data-titulaire", "data"), + Input("btn-download-data-titulaire", "n_clicks"), State(component_id="titulaire_data", component_property="data"), State(component_id="titulaire_nom", component_property="children"), State(component_id="titulaire_year", component_property="value"), @@ -273,3 +290,39 @@ def download_titulaire_data( date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return dcc.send_bytes(to_bytes, filename=f"decp_{titulaire_nom}_{date}.xlsx") + + +@callback( + Output("titulaire-download-filtered-data", "data"), + State("titulaire_data", "data"), + Input("btn-download-filtered-data-titulaire", "n_clicks"), + State("titulaire_nom", "children"), + State("titulaire_datatable", "filter_query"), + State("titulaire_datatable", "sort_by"), + State("titulaire_datatable", "hidden_columns"), + prevent_initial_call=True, +) +def download_filtered_titulaire_data( + data, n_clicks, titulaire_nom, filter_query, sort_by, hidden_columns: list = None +): + lff: pl.LazyFrame = pl.LazyFrame( + data + ) # start from the full titulaire data, not from paginated table data + + # Les colonnes masquées sont supprimées + if hidden_columns: + lff = lff.drop(hidden_columns) + + if filter_query: + lff = filter_table_data(lff, filter_query) + + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) + + def to_bytes(buffer): + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes( + to_bytes, filename=f"decp_filtrées_{titulaire_nom}_{date}.xlsx" + ) diff --git a/src/utils.py b/src/utils.py index d520f74..fecbd76 100644 --- a/src/utils.py +++ b/src/utils.py @@ -330,9 +330,10 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup continue column_object = data_schema.get(column_id) if column_object: - column_name = column_object.get("title", column_id) + column_name = column_object.get("title") else: column_name = column_id + print("Colonne inconnue dans le schéma !", column_id) column = { "name": column_name, @@ -588,16 +589,15 @@ def prepare_table_data( # Formatage des montants dff = format_values(dff) + + # Récupération des colonnes et tooltip columns, tooltip = setup_table_columns(dff) + dicts = dff.to_dicts() - if height > 65000: - download_disabled = True - download_text = "Téléchargement désactivé au-delà de 65 000 lignes" - download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." - else: - download_disabled = False - download_text = "Télécharger au format Excel" - download_title = "" + + # Propriétés du bouton de téléchargement + download_disabled, download_text, download_title = get_button_properties(height) + return ( dicts, columns, @@ -610,6 +610,19 @@ def prepare_table_data( ) +def get_button_properties(height): + if height > 65000: + download_disabled = True + download_text = "Téléchargement désactivé au-delà de 65 000 lignes" + download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution." + else: + print("moins de 65k") + download_disabled = False + download_text = "Télécharger au format Excel" + download_title = "" + return download_disabled, download_text, download_title + + df: pl.DataFrame = get_decp_data() schema = df.collect_schema()