From 636e0b059dc8b804dd19098134af1ca91d656586 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 21 Aug 2025 11:56:39 +0200 Subject: [PATCH 01/14] =?UTF-8?q?R=C3=A9cup=C3=A9ration=20des=20donn=C3=A9?= =?UTF-8?q?es=20acheteur=20depuis=20annuaire=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home.py | 4 ---- src/utils.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index c7b4095..ac9c0e8 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -21,10 +21,6 @@ update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") df_filtered = pl.DataFrame() -# Unique les données actuelles, pas les anciennes versions de marchés - -lf = lf.filter(pl.col("donneesActuelles")) - # Suppression des colonnes inutiles lf = lf.drop( [ diff --git a/src/utils.py b/src/utils.py index ace70c8..1be984d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,6 +5,7 @@ from time import sleep import polars as pl import polars.selectors as cs from dotenv import load_dotenv +from httpx import get from polars.exceptions import ComputeError load_dotenv() @@ -101,6 +102,12 @@ def format_number(number) -> str: return number +def get_acheteur_data(siret: str) -> dict: + url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" + response = get(url) + return response.json() + + def get_decp_data() -> pl.LazyFrame: # Chargement du fichier parquet # Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment. @@ -123,7 +130,10 @@ def get_decp_data() -> pl.LazyFrame: # lff = numbers_to_strings(lff) # Tri des marchés par date de notification - lff = lff.sort(by=["datePublicationDonnees"], descending=True, nulls_last=True) + lff = lff.sort(by=["dateNotification"], descending=True, nulls_last=True) + + # Uniquement les données actuelles, pas les anciennes versions de marchés + lff = lff.filter(pl.col("donneesActuelles")) return lff From 41d3ca75391666719ecbe99a3332801904871e8c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 19 Sep 2025 08:20:15 +0200 Subject: [PATCH 02/14] WIP #28 --- src/pages/acheteur.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/pages/acheteur.py diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py new file mode 100644 index 0000000..051d52b --- /dev/null +++ b/src/pages/acheteur.py @@ -0,0 +1,35 @@ +import dash +from dash import Input, Output, callback, dcc, html, register_page + +register_page( + __name__, + path_template="/acheteur/", + title="decp.info - acheteur", + name="Acheteur", + order=5, +) + +# 21690123100011 + +print(dash.page_registry["pages.acheteur"]) + +layout = [ + dcc.Location(id="url", refresh="callback-nav"), + html.Div( + className="container", + children=[ + html.H2(id="acheteur_title", children=""), + ], + ), +] + + +@callback( + Output(component_id="acheteur_title", component_property="children"), + Input(component_id="url", component_property="pathname"), +) +def update_acheteur(url): + acheteur_siret = url.split("/")[-1] + acheteur_title = acheteur_siret + + return acheteur_title From df04286a4aab6cf594f50d45abc0dd60f8016806 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 11:48:57 +0200 Subject: [PATCH 03/14] =?UTF-8?q?Derniers=20march=C3=A9=20notifi=C3=A9s=20?= =?UTF-8?q?#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/style.css | 5 ++- src/pages/acheteur.py | 69 ++++++++++++++++++++++++++++++--- src/pages/home.py | 89 ++++++++++++++++++++++--------------------- 3 files changed, 113 insertions(+), 50 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 2532b92..88f1e3f 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -71,7 +71,10 @@ td[data-dash-column="objet"] { } /* Alternance des couleurs pour les lignes */ -#table tr:nth-child(even) td { +.marches_table { + font-family: sans-serif; +} +.marches_table .cell-table tr:nth-child(even) td { background-color: #feeeee; } diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 051d52b..f767e6b 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,5 +1,7 @@ -import dash -from dash import Input, Output, callback, dcc, html, register_page +import polars as pl +from dash import Input, Output, callback, dash_table, dcc, html, register_page + +from src.utils import lf register_page( __name__, @@ -11,14 +13,15 @@ register_page( # 21690123100011 -print(dash.page_registry["pages.acheteur"]) - layout = [ + dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), html.Div( className="container", children=[ html.H2(id="acheteur_title", children=""), + html.H3("Derniers marchés publics notifiés"), + html.Div(id="acheteur_last_marches", children=""), ], ), ] @@ -30,6 +33,60 @@ layout = [ ) def update_acheteur(url): acheteur_siret = url.split("/")[-1] - acheteur_title = acheteur_siret + if len(acheteur_siret) != 14: + return f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" - return acheteur_title + return acheteur_siret + + +@callback( + Output(component_id="acheteur_data", component_property="data"), + Input(component_id="url", component_property="pathname"), +) +def get_acheteur_marches_data(url) -> pl.LazyFrame: + acheteur_siret = url.split("/")[-1] + lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + lff = lff.select( + "uid", + "objet", + "dateNotification", + "titulaire_nom", + "montant", + "codeCPV", + "dureeMois", + ) + lff = lff.sort("dateNotification", descending=True, nulls_last=True) + data = lff.collect(engine="streaming").to_dicts() + return data + + +@callback( + Output(component_id="acheteur_last_marches", component_property="children"), + Input(component_id="acheteur_data", component_property="data"), +) +def get_last_marches_table(data) -> html.Div: + table = html.Div( + className="marches_table", + children=dash_table.DataTable( + data=data[:20], + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "300px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "titulaire_nom"}, + "minWidth": "200px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + ), + ) + return table diff --git a/src/pages/home.py b/src/pages/home.py index 31c1ca4..780bfad 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -47,49 +47,52 @@ schema = lf.collect_schema() title = "Tableau" register_page(__name__, path="/", title="decp.info", name=title, order=1) -datatable = dash_table.DataTable( - cell_selectable=False, - id="table", - page_size=20, - page_current=0, - page_action="custom", - filter_action="custom", - filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": True, - } - for i in lf.collect_schema().names() - ], - sort_action="custom", - sort_mode="multi", - sort_by=[], - row_deletable=False, - style_cell_conditional=[ - { - "if": {"column_id": "objet"}, - "minWidth": "350px", - "textAlign": "left", - "overflow": "hidden", - "lineHeight": "14px", - "whiteSpace": "normal", - }, - { - "if": {"column_id": "acheteur_nom"}, - "minWidth": "250px", - "textAlign": "left", - "overflow": "hidden", - "lineHeight": "14px", - "whiteSpace": "normal", - }, - ], - data_timestamp=0, - markdown_options={"html": True}, +datatable = html.Div( + className="marches_table", + children=dash_table.DataTable( + cell_selectable=False, + id="table", + page_size=20, + page_current=0, + page_action="custom", + filter_action="custom", + filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, + columns=[ + { + "name": i, + "id": i, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": True, + } + for i in lf.collect_schema().names() + ], + sort_action="custom", + sort_mode="multi", + sort_by=[], + row_deletable=False, + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "350px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "acheteur_nom"}, + "minWidth": "250px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + data_timestamp=0, + markdown_options={"html": True}, + ), ) layout = [ From 59321fc80ac66eef25484aed3f644e6c8b205343 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 14:29:48 +0200 Subject: [PATCH 04/14] Commune acheteur, carte localisation #28 --- src/assets/style.css | 17 ++++++++++++++++ src/figures.py | 47 ++++++++++++++++++++++++++++++++++++++----- src/pages/acheteur.py | 43 ++++++++++++++++++++++++++++++++++----- src/utils.py | 4 ++-- 4 files changed, 99 insertions(+), 12 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 88f1e3f..2058973 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -111,3 +111,20 @@ h3 { #_pages_content { padding-top: 28px; } + +/* Vue acheteur/fournisseur */ +.wrapper { + display: grid; + grid-gap: 10px; + margin-bottom: 50px; +} + +.org_infos { + grid-column: 1; + grid-row: 1; +} + +.org_map { + grid-column: 2; + grid-row: 1; +} diff --git a/src/figures.py b/src/figures.py index 8ca0c5d..9be7fa6 100644 --- a/src/figures.py +++ b/src/figures.py @@ -2,7 +2,7 @@ import json import plotly.express as px import polars as pl -from dash import dash_table, html +from dash import dash_table, dcc, html def get_map_count_marches(lf: pl.LazyFrame): @@ -32,11 +32,11 @@ def get_map_count_marches(lf: pl.LazyFrame): df, geojson=departements, locations="Département", - color="uid", - color_continuous_scale="Reds", + # color="uid", + # color_continuous_scale="Reds", title="Nombres de marchés attribués par département (lieu d'exécution)", - range_color=(df["uid"].min(), df["uid"].max()), - labels={"uid": "Marchés attribués"}, + # range_color=(df["uid"].min(), df["uid"].max()), + # labels={"uid": "Marchés attribués"}, scope="europe", width=1000, height=800, @@ -157,3 +157,40 @@ def get_sources_tables(source_path) -> html.Div: datatable.data = df.to_dicts() return html.Div(children=datatable) + + +def point_on_map(lat, lon): + lat = float(lat) + lon = float(lon) + + # Create a scatter mapbox or choropleth map + fig = px.scatter_map( + lat=[lat], + lon=[lon], + height=300, + width=400, + color=[1], + ) + + fig.update_coloraxes(showscale=False) + + # Set map style (you can use 'open-street-map', 'carto-positron', etc.) + fig.update_layout( + mapbox_style="light", # Light, clean background + margin={"r": 0, "t": 0, "l": 0, "b": 0}, + ) + + # Optionally, center the map on France + fig.update_geos( + center=dict(lat=46.603354, lon=1.888334), # Center of France + lataxis_range=[41, 51.5], # Latitude range for France + lonaxis_range=[-5, 10], # Longitude range for France + ) + + # But scatter_mapbox doesn't use geos, so better to control via zoom/center manually + # Let's reset and use proper centering in scatter_mapbox instead: + + fig.update_layout(map_center={"lat": 46.6, "lon": 1.89}, map_zoom=4) + + graph = dcc.Graph(id="map", figure=fig) + return graph diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index f767e6b..5b14664 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,7 +1,8 @@ import polars as pl from dash import Input, Output, callback, dash_table, dcc, html, register_page -from src.utils import lf +from src.figures import point_on_map +from src.utils import get_annuaire_data, lf register_page( __name__, @@ -19,7 +20,27 @@ layout = [ html.Div( className="container", children=[ - html.H2(id="acheteur_title", children=""), + html.H2( + children=[ + html.Span(id="acheteur_siret"), + " - ", + html.Span(id="acheteur_nom"), + ] + ), + html.Div( + className="wrapper", + children=[ + html.Div( + className="org_infos", + children=[html.P(["Commune : ", html.Span(id="commune")])], + ), + html.Div(className="org_map", id="acheteur_map"), + # adresse + # code commune + # lat long + ], + ), + # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics notifiés"), html.Div(id="acheteur_last_marches", children=""), ], @@ -28,15 +49,27 @@ layout = [ @callback( - Output(component_id="acheteur_title", component_property="children"), + Output(component_id="acheteur_siret", component_property="children"), + Output(component_id="acheteur_nom", component_property="children"), + Output(component_id="commune", component_property="children"), + Output(component_id="acheteur_map", component_property="children"), Input(component_id="url", component_property="pathname"), ) def update_acheteur(url): acheteur_siret = url.split("/")[-1] if len(acheteur_siret) != 14: return f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" - - return acheteur_siret + data = get_annuaire_data(acheteur_siret) + data_etablissement = data["matching_etablissements"][0] + acheteur_map = point_on_map( + data_etablissement["latitude"], data_etablissement["longitude"] + ) + return ( + acheteur_siret, + data["nom_raison_sociale"], + data_etablissement["libelle_commune"], + acheteur_map, + ) @callback( diff --git a/src/utils.py b/src/utils.py index 3286b15..cc49a09 100644 --- a/src/utils.py +++ b/src/utils.py @@ -102,10 +102,10 @@ def format_number(number) -> str: return number -def get_acheteur_data(siret: str) -> dict: +def get_annuaire_data(siret: str) -> dict: url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" response = get(url) - return response.json() + return response.json()["results"][0] def get_decp_data() -> pl.LazyFrame: From b88d1c9f9382a913e8ff2469f91482f9ae8213f9 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 18:16:53 +0200 Subject: [PATCH 05/14] =?UTF-8?q?Pagination=20derniers=20march=C3=A9s=20#2?= =?UTF-8?q?8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 5b14664..7db212b 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -101,7 +101,9 @@ def get_last_marches_table(data) -> html.Div: table = html.Div( className="marches_table", children=dash_table.DataTable( - data=data[:20], + data=data, + page_action="native", + page_size=10, style_cell_conditional=[ { "if": {"column_id": "objet"}, From 2fe92d8a1475c1c8103078bda1ec83f88775ba3d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 20:49:15 +0200 Subject: [PATCH 06/14] Plus d'informations, stats #28 --- src/assets/style.css | 27 ++++++++- src/pages/acheteur.py | 129 ++++++++++++++++++++++++++++++++++++------ src/utils.py | 21 ++++++- 3 files changed, 155 insertions(+), 22 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 2058973..c8449d4 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -76,6 +76,7 @@ td[data-dash-column="objet"] { } .marches_table .cell-table tr:nth-child(even) td { background-color: #feeeee; + font-family: sans-serif; } #header > *, @@ -117,14 +118,34 @@ h3 { display: grid; grid-gap: 10px; margin-bottom: 50px; + justify-content: space-between; +} + +.org_title { + grid-column: 1 / 3; + grid-row: 1; +} + +.org_year { + grid-column: 3; + grid-row: 1; } .org_infos { grid-column: 1; - grid-row: 1; + grid-row: 2; +} + +.org_infos > p { + margin: 8px 0; +} + +.org_stats { + grid-column: 2; + grid-row: 2; } .org_map { - grid-column: 2; - grid-row: 1; + grid-column: 3; + grid-row: 2; } diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 7db212b..56a50a9 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,8 +1,10 @@ +import datetime + import polars as pl from dash import Input, Output, callback, dash_table, dcc, html, register_page from src.figures import point_on_map -from src.utils import get_annuaire_data, lf +from src.utils import format_number, get_annuaire_data, get_departement_region, lf register_page( __name__, @@ -20,24 +22,58 @@ layout = [ html.Div( className="container", children=[ - html.H2( - children=[ - html.Span(id="acheteur_siret"), - " - ", - html.Span(id="acheteur_nom"), - ] - ), html.Div( className="wrapper", children=[ + html.H2( + className="org_title", + children=[ + html.Span(id="acheteur_siret"), + " - ", + html.Span(id="acheteur_nom"), + ], + ), + html.Div( + className="org_year", + children=dcc.Dropdown( + id="acheteur_year", + options=["Toutes"] + + [ + str(year) + for year in range( + 2018, int(datetime.date.today().year) + 1 + ) + ], + placeholder="Année", + ), + ), html.Div( className="org_infos", - children=[html.P(["Commune : ", html.Span(id="commune")])], + children=[ + html.P(["Commune : ", html.Strong(id="acheteur_commune")]), + html.P( + [ + "Département : ", + html.Strong(id="acheteur_departement"), + ] + ), + html.P(["Région : ", html.Strong(id="acheteur_region")]), + html.A( + id="acheteur_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + target="_blank", + ), + ], + ), + html.Div( + className="org_stats", + children=[ + html.P(id="acheteur_titre_stats"), + html.P(id="acheteur_marches_notifies"), + html.P(id="acheteur_fournisseurs_differents"), + ], ), html.Div(className="org_map", id="acheteur_map"), - # adresse - # code commune - # lat long ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -51,38 +87,85 @@ layout = [ @callback( Output(component_id="acheteur_siret", component_property="children"), Output(component_id="acheteur_nom", component_property="children"), - Output(component_id="commune", component_property="children"), + Output(component_id="acheteur_commune", component_property="children"), Output(component_id="acheteur_map", component_property="children"), + Output(component_id="acheteur_departement", component_property="children"), + Output(component_id="acheteur_region", component_property="children"), + Output(component_id="acheteur_lien_annuaire", component_property="href"), Input(component_id="url", component_property="pathname"), ) -def update_acheteur(url): +def update_acheteur_infos(url): acheteur_siret = url.split("/")[-1] if len(acheteur_siret) != 14: - return f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" + acheteur_siret = ( + f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" + ) data = get_annuaire_data(acheteur_siret) data_etablissement = data["matching_etablissements"][0] acheteur_map = point_on_map( data_etablissement["latitude"], data_etablissement["longitude"] ) + code_departement, nom_departement, nom_region = get_departement_region( + data_etablissement["code_postal"] + ) + departement = f"{nom_departement} ({code_departement})" + lien_annuaire = ( + f"https://annuaire-entreprises.data.gouv.fr/etablissement/{acheteur_siret}" + ) return ( acheteur_siret, data["nom_raison_sociale"], data_etablissement["libelle_commune"], acheteur_map, + departement, + nom_region, + lien_annuaire, ) +@callback( + Output(component_id="acheteur_marches_notifies", component_property="children"), + Output( + component_id="acheteur_fournisseurs_differents", component_property="children" + ), + Input(component_id="acheteur_data", component_property="data"), +) +def update_acheteur_stats(data): + df = pl.DataFrame(data) + df_marches = df.unique("uid") + nb_marches = format_number(df_marches.height) + # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) + marches_notifies = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] + # + ", pour un total de ", html.Strong(somme_marches + " €")] + del df_marches + + nb_fournisseurs = df.unique("titulaire_id").height + nb_fournisseurs = [ + html.Strong(format_number(nb_fournisseurs)), + " fournisseurs (SIRET) différents", + ] + del df + + return marches_notifies, nb_fournisseurs + + @callback( Output(component_id="acheteur_data", component_property="data"), Input(component_id="url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url) -> pl.LazyFrame: +def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: acheteur_siret = url.split("/")[-1] lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + if acheteur_year and acheteur_year != "Toutes": + lff = lff.filter( + pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) + ) lff = lff.select( "uid", "objet", "dateNotification", + "titulaire_id", "titulaire_nom", "montant", "codeCPV", @@ -98,11 +181,25 @@ def get_acheteur_marches_data(url) -> pl.LazyFrame: Input(component_id="acheteur_data", component_property="data"), ) def get_last_marches_table(data) -> html.Div: + columns = data[0].keys() + table = html.Div( className="marches_table", children=dash_table.DataTable( data=data, page_action="native", + columns=[ + { + "name": i, + "id": i, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": False, + } + for i in columns + if i not in ["titulaire_id"] + ], page_size=10, style_cell_conditional=[ { diff --git a/src/utils.py b/src/utils.py index cc49a09..9900ee9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,3 +1,4 @@ +import json import logging import os from time import sleep @@ -124,9 +125,6 @@ def get_decp_data() -> pl.LazyFrame: sleep(10) lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH")) - # Remplacement des valeurs numériques par des chaînes de caractères - # lff = numbers_to_strings(lff) - # Tri des marchés par date de notification lff = lff.sort(by=["dateNotification"], descending=True, nulls_last=True) @@ -136,4 +134,21 @@ def get_decp_data() -> pl.LazyFrame: return lff +def get_departements() -> dict: + with open("data/departements.json", "rb") as f: + data = json.load(f) + return data + + +def get_departement_region(code_postal): + if code_postal > "97000": + code_departement = code_postal[:3] + else: + code_departement = code_postal[:2] + nom_departement = departements[code_departement]["departement"] + nom_region = departements[code_departement]["region"] + return code_departement, nom_departement, nom_region + + lf = get_decp_data() +departements = get_departements() From 9e5673c28f8418adde4187f9e377c9b7fef2ec0a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 21:14:11 +0200 Subject: [PATCH 07/14] Ajout des liens dans le tableau le plus tard possible --- src/pages/home.py | 48 ++++++++++++++++++++++------------------------- src/utils.py | 16 ++++++++-------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index 780bfad..971fff3 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -34,13 +34,6 @@ lf = booleans_to_strings(lf) # Remplacer les valeurs manquantes par des chaînes vides lf = lf.fill_null("") - -# Ajout des liens vers l'annuaire -lf = add_annuaire_link(lf) - -# Ajout des liens open data -lf = add_resource_link(lf) - schema = lf.collect_schema() @@ -57,17 +50,6 @@ datatable = html.Div( page_action="custom", filter_action="custom", filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": True, - } - for i in lf.collect_schema().names() - ], sort_action="custom", sort_mode="multi", sort_by=[], @@ -158,6 +140,7 @@ layout = [ @callback( Output("table", "data"), + Output("table", "columns"), Output("table", "data_timestamp"), Output("nb_rows", "children"), Output("btn-download-data", "disabled"), @@ -229,6 +212,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) start_row = page_current * page_size # end_row = (page_current + 1) * page_size dff = dff.slice(start_row, page_size) + + # Ajout des liens vers l'annuaire des entreprises + dff = add_annuaire_link(dff) + + # Ajout des liens vers les fichiers Open Data + dff = add_resource_link(dff) + + # Liste finale de colonnes + columns = [ + { + "name": column, + "id": column, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": True, + } + for column in dff.columns + ] + dicts = dff.to_dicts() if height > 65000: @@ -242,6 +245,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) return ( dicts, + columns, data_timestamp + 1, nb_rows, download_disabled, @@ -259,14 +263,6 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) def download_data(n_clicks, hidden_columns: list = None): df_to_download = df_filtered.clone() - print(df_to_download.columns) - - # Rétablissement des colonnes source et sourceOpenData (voir add_resource_link) - df_to_download = df_to_download.with_columns( - pl.col("source").str.extract(r'href="(.*?)"').alias("sourceFile"), - pl.col("source").str.extract(r'">(.*?)<').alias("sourceDataset"), - ) - # Les colonnes masquées sont supprimées if hidden_columns: df_to_download = df_to_download.drop(hidden_columns) diff --git a/src/utils.py b/src/utils.py index 9900ee9..a7ef63c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -42,18 +42,18 @@ def split_filter_part(filter_part): return [None] * 3 -def add_resource_link(lff: pl.LazyFrame) -> pl.LazyFrame: - lff = lff.with_columns( +def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame: + dff = dff.with_columns( ( '' + pl.col("sourceDataset") + "" ).alias("source") ) - lff = lff.drop(["sourceFile", "sourceDataset"]) - return lff + dff = dff.drop(["sourceFile", "sourceDataset"]) + return dff -def add_annuaire_link(lff: pl.LazyFrame): - lff = lff.with_columns( +def add_annuaire_link(dff: pl.DataFrame): + dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( '" ).alias("acheteur_id") ) - return lff + return dff def booleans_to_strings(lff: pl.LazyFrame) -> pl.LazyFrame: From e88992a470eb579138a800c24d5d78817f15258a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 21:39:06 +0200 Subject: [PATCH 08/14] =?UTF-8?q?Le=20lien=20acheteur=20m=C3=A8ne=20mainte?= =?UTF-8?q?nant=20=C3=A0=20la=20page=20acheteur=20au=20lieu=20de=20l'annua?= =?UTF-8?q?ire=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app.py | 1 + src/figures.py | 6 +----- src/pages/home.py | 4 ++-- src/utils.py | 6 +++--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/app.py b/src/app.py index e45d9ef..66e5f30 100644 --- a/src/app.py +++ b/src/app.py @@ -74,6 +74,7 @@ app.layout = html.Div( page["name"], href=page["relative_path"], className="nav" ) for page in page_registry.values() + if page["name"] not in ["Acheteur", "Fournisseur"] ] ), ], diff --git a/src/figures.py b/src/figures.py index 9be7fa6..3361e06 100644 --- a/src/figures.py +++ b/src/figures.py @@ -165,11 +165,7 @@ def point_on_map(lat, lon): # Create a scatter mapbox or choropleth map fig = px.scatter_map( - lat=[lat], - lon=[lon], - height=300, - width=400, - color=[1], + lat=[lat], lon=[lon], height=300, width=400, color=[1], size=[1] ) fig.update_coloraxes(showscale=False) diff --git a/src/pages/home.py b/src/pages/home.py index 971fff3..3377440 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -6,7 +6,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from dotenv import load_dotenv from src.utils import ( - add_annuaire_link, + add_org_links, add_resource_link, booleans_to_strings, format_number, @@ -214,7 +214,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) dff = dff.slice(start_row, page_size) # Ajout des liens vers l'annuaire des entreprises - dff = add_annuaire_link(dff) + dff = add_org_links(dff) # Ajout des liens vers les fichiers Open Data dff = add_resource_link(dff) diff --git a/src/utils.py b/src/utils.py index a7ef63c..f7df698 100644 --- a/src/utils.py +++ b/src/utils.py @@ -52,7 +52,7 @@ def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame: return dff -def add_annuaire_link(dff: pl.DataFrame): +def add_org_links(dff: pl.DataFrame): dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( @@ -67,9 +67,9 @@ def add_annuaire_link(dff: pl.DataFrame): ) dff = dff.with_columns( ( - '' + + '" target="_blank">' + pl.col("acheteur_id") + "" ).alias("acheteur_id") From 8f335a52ea91a59d69db09bebea9be24f81df8e5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 21:50:33 +0200 Subject: [PATCH 09/14] =?UTF-8?q?Notifi=C3=A9=20=3D>=20attribu=C3=A9=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 56a50a9..f47eca6 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -69,7 +69,7 @@ layout = [ className="org_stats", children=[ html.P(id="acheteur_titre_stats"), - html.P(id="acheteur_marches_notifies"), + html.P(id="acheteur_marches_attribues"), html.P(id="acheteur_fournisseurs_differents"), ], ), @@ -77,7 +77,7 @@ layout = [ ], ), # récupérer les données de l'acheteur sur l'api annuaire - html.H3("Derniers marchés publics notifiés"), + html.H3("Derniers marchés publics attribués"), html.Div(id="acheteur_last_marches", children=""), ], ), @@ -124,7 +124,7 @@ def update_acheteur_infos(url): @callback( - Output(component_id="acheteur_marches_notifies", component_property="children"), + Output(component_id="acheteur_marches_attribues", component_property="children"), Output( component_id="acheteur_fournisseurs_differents", component_property="children" ), @@ -135,7 +135,7 @@ def update_acheteur_stats(data): df_marches = df.unique("uid") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) - marches_notifies = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] + marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] # + ", pour un total de ", html.Strong(somme_marches + " €")] del df_marches @@ -146,7 +146,7 @@ def update_acheteur_stats(data): ] del df - return marches_notifies, nb_fournisseurs + return marches_attribues, nb_fournisseurs @callback( From 944f207d9b1b949427f64db5cc381bbef585ea50 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 23:15:20 +0200 Subject: [PATCH 10/14] =?UTF-8?q?T=C3=A9l=C3=A9chargement=20des=20donn?= =?UTF-8?q?=C3=A9es=20acheteur=20filtr=C3=A9es=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 60 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index f47eca6..ded95ea 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -71,6 +71,12 @@ layout = [ html.P(id="acheteur_titre_stats"), html.P(id="acheteur_marches_attribues"), html.P(id="acheteur_fournisseurs_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-acheteur-data", + ), + dcc.Download(id="download-acheteur-data"), + dcc.Store(id="n_acheteur_downloads"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -132,6 +138,8 @@ def update_acheteur_infos(url): ) def update_acheteur_stats(data): df = pl.DataFrame(data) + if df.height == 0: + df = pl.DataFrame(schema=lf.collect_schema()) df_marches = df.unique("uid") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) @@ -157,10 +165,6 @@ def update_acheteur_stats(data): def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: acheteur_siret = url.split("/")[-1] lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) - if acheteur_year and acheteur_year != "Toutes": - lff = lff.filter( - pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) - ) lff = lff.select( "uid", "objet", @@ -171,7 +175,12 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: "codeCPV", "dureeMois", ) + if acheteur_year and acheteur_year != "Toutes": + lff = lff.filter( + pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) + ) lff = lff.sort("dateNotification", descending=True, nulls_last=True) + data = lff.collect(engine="streaming").to_dicts() return data @@ -181,10 +190,19 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: Input(component_id="acheteur_data", component_property="data"), ) def get_last_marches_table(data) -> html.Div: - columns = data[0].keys() + columns = [ + "uid", + "objet", + "dateNotification", + "titulaire_nom", + "montant", + "codeCPV", + "dureeMois", + ] table = html.Div( className="marches_table", + id="marches_datatable", children=dash_table.DataTable( data=data, page_action="native", @@ -222,3 +240,35 @@ def get_last_marches_table(data) -> html.Div: ), ) return table + + +@callback( + Output("download-acheteur-data", "data"), + Output("n_acheteur_downloads", "data"), + Input("btn-download-acheteur-data", "n_clicks"), + Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_nom", component_property="children"), + Input(component_id="acheteur_year", component_property="value"), + Input("n_acheteur_downloads", "data"), + prevent_initial_call=True, +) +def download_acheteur_data( + n_clicks, data: [dict], acheteur_nom: str, annee: str, n_downloads: int +): + if n_clicks is None: + return None, 0 + if n_clicks == n_downloads: + return None, n_downloads + + df_to_download = pl.DataFrame(data) + + def to_bytes(buffer): + df_to_download.write_excel( + buffer, worksheet="DECP" if annee in ["Toutes", None] else annee + ) + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + n_downloads += 1 + return dcc.send_bytes( + to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx" + ), n_downloads From bfa181cefd9de01dca6b1c3632eb9dfa5c5a251b Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 23:57:51 +0200 Subject: [PATCH 11/14] Suppression de la variable globale df_filtered, et adaptations --- src/pages/home.py | 70 ++++++++++++++++------------------------------- src/utils.py | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index 3377440..81ad241 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -9,17 +9,16 @@ from src.utils import ( add_org_links, add_resource_link, booleans_to_strings, + filter_table_data, format_number, lf, - logger, - split_filter_part, + sort_table_data, ) load_dotenv() update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -df_filtered = pl.DataFrame() # Suppression des colonnes inutiles lf = lf.drop( @@ -128,6 +127,7 @@ layout = [ disabled=True, ), dcc.Download(id="download-data"), + dcc.Store(id="filtered_data", storage_type="memory"), html.P("Données mises à jour le " + str(update_date)), ], className="table-menu", @@ -141,6 +141,7 @@ layout = [ @callback( Output("table", "data"), Output("table", "columns"), + # Output("filtered_data", "data"), Output("table", "data_timestamp"), Output("nb_rows", "children"), Output("btn-download-data", "disabled"), @@ -154,55 +155,22 @@ layout = [ ) def update_table(page_current, page_size, filter_query, sort_by, data_timestamp): print(" + + + + + + + + + + + + + + + + + + ") - global df_filtered # Application des filtres lff: pl.LazyFrame = lf # start from the original data if filter_query: - filtering_expressions = filter_query.split(" && ") - for filter_part in filtering_expressions: - col_name, operator, filter_value = split_filter_part(filter_part) - col_type = str(schema[col_name]) - print("filter_value:", filter_value) - print("filter_value_type:", type(filter_value)) - print("col_type:", col_type) - - if operator in ("<", "<=", ">", ">="): - filter_value = int(filter_value) - if operator == "<": - lff = lff.filter(pl.col(col_name) < filter_value) - elif operator == ">": - lff = lff.filter(pl.col(col_name) > filter_value) - elif operator == ">=": - lff = lff.filter(pl.col(col_name) >= filter_value) - elif operator == "<=": - lff = lff.filter(pl.col(col_name) <= filter_value) - - elif col_type.startswith("Int") or col_type.startswith("Float"): - try: - filter_value = int(filter_value) - except ValueError: - logger.error(f"Invalid numeric filter value: {filter_value}") - continue - lff = lff.filter(pl.col(col_name) == filter_value) - - elif operator == "contains" and col_type == "String": - lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) - - # elif operator == 'datestartswith': - # lff = lff.filter(pl.col(col_name).str.startswith(filter_value)") + lff = filter_table_data(lff, filter_query) if len(sort_by) > 0: - lff = lff.sort( - [col["column_id"] for col in sort_by], - descending=[col["direction"] == "desc" for col in sort_by], - nulls_last=True, - ) - print(sort_by) + lff = sort_table_data(lff, sort_by) + # Matérialisation des filtres dff: pl.DataFrame = lff.collect() - df_filtered = dff.clone() + # if filter_query or sort_by: + # filtered_data = dff.to_dicts() + # else: + # filtered_data = {} height = dff.height @@ -257,18 +225,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) @callback( Output("download-data", "data"), Input("btn-download-data", "n_clicks"), + State("table", "filter_query"), + State("table", "sort_by"), State("table", "hidden_columns"), prevent_initial_call=True, ) -def download_data(n_clicks, hidden_columns: list = None): - df_to_download = df_filtered.clone() +def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None): + lff: pl.LazyFrame = lf # start from the original data # Les colonnes masquées sont supprimées if hidden_columns: - df_to_download = df_to_download.drop(hidden_columns) + lff = lff.drop(hidden_columns) + + if filter_query: + lff = filter_table_data(lff, filter_query) + + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) def to_bytes(buffer): - df_to_download.write_excel(buffer, worksheet="DECP") + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return dcc.send_bytes(to_bytes, filename=f"decp_{date}.xlsx") diff --git a/src/utils.py b/src/utils.py index f7df698..1cd916c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -150,5 +150,53 @@ def get_departement_region(code_postal): return code_departement, nom_departement, nom_region +def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: + schema = lff.collect_schema() + filtering_expressions = filter_query.split(" && ") + for filter_part in filtering_expressions: + col_name, operator, filter_value = split_filter_part(filter_part) + col_type = str(schema[col_name]) + print("filter_value:", filter_value) + print("filter_value_type:", type(filter_value)) + print("col_type:", col_type) + + if operator in ("<", "<=", ">", ">="): + filter_value = int(filter_value) + if operator == "<": + lff = lff.filter(pl.col(col_name) < filter_value) + elif operator == ">": + lff = lff.filter(pl.col(col_name) > filter_value) + elif operator == ">=": + lff = lff.filter(pl.col(col_name) >= filter_value) + elif operator == "<=": + lff = lff.filter(pl.col(col_name) <= filter_value) + + elif col_type.startswith("Int") or col_type.startswith("Float"): + try: + filter_value = int(filter_value) + except ValueError: + logger.error(f"Invalid numeric filter value: {filter_value}") + continue + lff = lff.filter(pl.col(col_name) == filter_value) + + elif operator == "contains" and col_type == "String": + lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) + + # elif operator == 'datestartswith': + # lff = lff.filter(pl.col(col_name).str.startswith(filter_value)") + + return lff + + +def sort_table_data(lff: pl.LazyFrame, sort_by: list) -> pl.LazyFrame: + lff = lff.sort( + [col["column_id"] for col in sort_by], + descending=[col["direction"] == "desc" for col in sort_by], + nulls_last=True, + ) + print(sort_by) + return lff + + lf = get_decp_data() departements = get_departements() From 1c5868d75fd5f50509f7d18540cf109d4e032dec Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 23:58:23 +0200 Subject: [PATCH 12/14] =?UTF-8?q?Utilisation=20de=20State=20plut=C3=B4t=20?= =?UTF-8?q?qu'Input=20pour=20ne=20pas=20trigger=20le=20download=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index ded95ea..965ccdd 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,7 +1,7 @@ import datetime import polars as pl -from dash import Input, Output, callback, dash_table, dcc, html, register_page +from dash import Input, Output, State, callback, dash_table, dcc, html, register_page from src.figures import point_on_map from src.utils import format_number, get_annuaire_data, get_departement_region, lf @@ -50,6 +50,7 @@ layout = [ html.Div( className="org_infos", children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. html.P(["Commune : ", html.Strong(id="acheteur_commune")]), html.P( [ @@ -76,7 +77,6 @@ layout = [ id="btn-download-acheteur-data", ), dcc.Download(id="download-acheteur-data"), - dcc.Store(id="n_acheteur_downloads"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -244,22 +244,18 @@ def get_last_marches_table(data) -> html.Div: @callback( Output("download-acheteur-data", "data"), - Output("n_acheteur_downloads", "data"), Input("btn-download-acheteur-data", "n_clicks"), - Input(component_id="acheteur_data", component_property="data"), - Input(component_id="acheteur_nom", component_property="children"), - Input(component_id="acheteur_year", component_property="value"), - Input("n_acheteur_downloads", "data"), + State(component_id="acheteur_data", component_property="data"), + State(component_id="acheteur_nom", component_property="children"), + State(component_id="acheteur_year", component_property="value"), prevent_initial_call=True, ) def download_acheteur_data( - n_clicks, data: [dict], acheteur_nom: str, annee: str, n_downloads: int + n_clicks, + data: [dict], + acheteur_nom: str, + annee: str, ): - if n_clicks is None: - return None, 0 - if n_clicks == n_downloads: - return None, n_downloads - df_to_download = pl.DataFrame(data) def to_bytes(buffer): @@ -268,7 +264,4 @@ def download_acheteur_data( ) date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") - n_downloads += 1 - return dcc.send_bytes( - to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx" - ), n_downloads + return dcc.send_bytes(to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx") From 98b8699afbc7ed5989bdc172f4e9f874dd78e3af Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 00:30:52 +0200 Subject: [PATCH 13/14] =?UTF-8?q?R=C3=A9tablissement=20de=20la=20carte=20d?= =?UTF-8?q?es=20march=C3=A9s=20par=20dpt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/style.css | 5 +++++ src/figures.py | 8 ++++---- src/pages/acheteur.py | 9 +++++---- src/pages/home.py | 22 +++------------------- src/utils.py | 11 +++++++++-- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index c8449d4..873998a 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -106,6 +106,11 @@ a.nav { } h3 { + margin: 36px 0 24px 0; +} + +summary > h3 { + margin: 0; display: inline; } diff --git a/src/figures.py b/src/figures.py index 3361e06..860c695 100644 --- a/src/figures.py +++ b/src/figures.py @@ -32,11 +32,11 @@ def get_map_count_marches(lf: pl.LazyFrame): df, geojson=departements, locations="Département", - # color="uid", - # color_continuous_scale="Reds", + color="uid", + color_continuous_scale="Reds", title="Nombres de marchés attribués par département (lieu d'exécution)", - # range_color=(df["uid"].min(), df["uid"].max()), - # labels={"uid": "Marchés attribués"}, + range_color=(df["uid"].min(), df["uid"].max()), + labels={"uid": "Marchés attribués"}, scope="europe", width=1000, height=800, diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 965ccdd..e3fa979 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -140,7 +140,7 @@ def update_acheteur_stats(data): df = pl.DataFrame(data) if df.height == 0: df = pl.DataFrame(schema=lf.collect_schema()) - df_marches = df.unique("uid") + df_marches = df.unique("id") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] @@ -165,8 +165,9 @@ def update_acheteur_stats(data): def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: acheteur_siret = url.split("/")[-1] lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + lff = lff.fill_null("") lff = lff.select( - "uid", + "id", "objet", "dateNotification", "titulaire_id", @@ -179,7 +180,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: lff = lff.filter( pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) ) - lff = lff.sort("dateNotification", descending=True, nulls_last=True) + lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) data = lff.collect(engine="streaming").to_dicts() return data @@ -191,7 +192,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: ) def get_last_marches_table(data) -> html.Div: columns = [ - "uid", + "id", "objet", "dateNotification", "titulaire_nom", diff --git a/src/pages/home.py b/src/pages/home.py index 81ad241..9e19d8c 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -8,7 +8,6 @@ from dotenv import load_dotenv from src.utils import ( add_org_links, add_resource_link, - booleans_to_strings, filter_table_data, format_number, lf, @@ -20,19 +19,6 @@ load_dotenv() update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -# Suppression des colonnes inutiles -lf = lf.drop( - [ - "donneesActuelles", - ] -) - -# Convertir les colonnes booléennes en chaînes de caractères -lf = booleans_to_strings(lf) - -# Remplacer les valeurs manquantes par des chaînes vides -lf = lf.fill_null("") - schema = lf.collect_schema() @@ -164,14 +150,12 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) if len(sort_by) > 0: lff = sort_table_data(lff, sort_by) + # Remplace les strings null par "", mais pas les numeric null + lff = lff.fill_null("") + # Matérialisation des filtres dff: pl.DataFrame = lff.collect() - # if filter_query or sort_by: - # filtered_data = dff.to_dicts() - # else: - # filtered_data = {} - height = dff.height nb_rows = f"{format_number(height)} lignes" diff --git a/src/utils.py b/src/utils.py index 1cd916c..96c2bfb 100644 --- a/src/utils.py +++ b/src/utils.py @@ -126,10 +126,17 @@ def get_decp_data() -> pl.LazyFrame: lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH")) # Tri des marchés par date de notification - lff = lff.sort(by=["dateNotification"], descending=True, nulls_last=True) + lff = lff.sort(by=["dateNotification", "uid"], descending=True, nulls_last=True) # Uniquement les données actuelles, pas les anciennes versions de marchés - lff = lff.filter(pl.col("donneesActuelles")) + lff = lff.filter(pl.col("donneesActuelles")).drop("donneesActuelles") + + # Convertir les colonnes booléennes en chaînes de caractères + lff = booleans_to_strings(lff) + + # Bizarrement je ne peux pas faire lff = lff.fill_null("") ici + # ça génère une erreur dans la page acheteur (acheteur_data.table) : + # AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import) return lff From 4b8565dcf2e11752a90acb6eb48db40ea772ec92 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 00:33:21 +0200 Subject: [PATCH 14/14] Chanlog --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7092317..70a2fae 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,12 @@ python run.py ## Notes de version -#### 2.0.1 (23 septembre 2025) +#### 2.1.0 + +- Ajout de la vue acheteur +- Variables globales uniquement en lecture + +##### 2.0.1 (23 septembre 2025) - Bloquage du bouton de téléchargement si trop de lignes (+ 65000) [#38](https://github.com/ColinMaudry/decp.info/issues/38) - Amélioration du script de déploiement (deploy.sh)