From 636e0b059dc8b804dd19098134af1ca91d656586 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 21 Aug 2025 11:56:39 +0200 Subject: [PATCH 01/41] =?UTF-8?q?R=C3=A9cup=C3=A9ration=20des=20donn=C3=A9?= =?UTF-8?q?es=20acheteur=20depuis=20annuaire=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home.py | 4 ---- src/utils.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index c7b4095..ac9c0e8 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -21,10 +21,6 @@ update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") df_filtered = pl.DataFrame() -# Unique les données actuelles, pas les anciennes versions de marchés - -lf = lf.filter(pl.col("donneesActuelles")) - # Suppression des colonnes inutiles lf = lf.drop( [ diff --git a/src/utils.py b/src/utils.py index ace70c8..1be984d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,6 +5,7 @@ from time import sleep import polars as pl import polars.selectors as cs from dotenv import load_dotenv +from httpx import get from polars.exceptions import ComputeError load_dotenv() @@ -101,6 +102,12 @@ def format_number(number) -> str: return number +def get_acheteur_data(siret: str) -> dict: + url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" + response = get(url) + return response.json() + + def get_decp_data() -> pl.LazyFrame: # Chargement du fichier parquet # Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment. @@ -123,7 +130,10 @@ def get_decp_data() -> pl.LazyFrame: # lff = numbers_to_strings(lff) # Tri des marchés par date de notification - lff = lff.sort(by=["datePublicationDonnees"], descending=True, nulls_last=True) + lff = lff.sort(by=["dateNotification"], descending=True, nulls_last=True) + + # Uniquement les données actuelles, pas les anciennes versions de marchés + lff = lff.filter(pl.col("donneesActuelles")) return lff From 41d3ca75391666719ecbe99a3332801904871e8c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 19 Sep 2025 08:20:15 +0200 Subject: [PATCH 02/41] WIP #28 --- src/pages/acheteur.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/pages/acheteur.py diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py new file mode 100644 index 0000000..051d52b --- /dev/null +++ b/src/pages/acheteur.py @@ -0,0 +1,35 @@ +import dash +from dash import Input, Output, callback, dcc, html, register_page + +register_page( + __name__, + path_template="/acheteur/", + title="decp.info - acheteur", + name="Acheteur", + order=5, +) + +# 21690123100011 + +print(dash.page_registry["pages.acheteur"]) + +layout = [ + dcc.Location(id="url", refresh="callback-nav"), + html.Div( + className="container", + children=[ + html.H2(id="acheteur_title", children=""), + ], + ), +] + + +@callback( + Output(component_id="acheteur_title", component_property="children"), + Input(component_id="url", component_property="pathname"), +) +def update_acheteur(url): + acheteur_siret = url.split("/")[-1] + acheteur_title = acheteur_siret + + return acheteur_title From df04286a4aab6cf594f50d45abc0dd60f8016806 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 11:48:57 +0200 Subject: [PATCH 03/41] =?UTF-8?q?Derniers=20march=C3=A9=20notifi=C3=A9s=20?= =?UTF-8?q?#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/style.css | 5 ++- src/pages/acheteur.py | 69 ++++++++++++++++++++++++++++++--- src/pages/home.py | 89 ++++++++++++++++++++++--------------------- 3 files changed, 113 insertions(+), 50 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 2532b92..88f1e3f 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -71,7 +71,10 @@ td[data-dash-column="objet"] { } /* Alternance des couleurs pour les lignes */ -#table tr:nth-child(even) td { +.marches_table { + font-family: sans-serif; +} +.marches_table .cell-table tr:nth-child(even) td { background-color: #feeeee; } diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 051d52b..f767e6b 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,5 +1,7 @@ -import dash -from dash import Input, Output, callback, dcc, html, register_page +import polars as pl +from dash import Input, Output, callback, dash_table, dcc, html, register_page + +from src.utils import lf register_page( __name__, @@ -11,14 +13,15 @@ register_page( # 21690123100011 -print(dash.page_registry["pages.acheteur"]) - layout = [ + dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), html.Div( className="container", children=[ html.H2(id="acheteur_title", children=""), + html.H3("Derniers marchés publics notifiés"), + html.Div(id="acheteur_last_marches", children=""), ], ), ] @@ -30,6 +33,60 @@ layout = [ ) def update_acheteur(url): acheteur_siret = url.split("/")[-1] - acheteur_title = acheteur_siret + if len(acheteur_siret) != 14: + return f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" - return acheteur_title + return acheteur_siret + + +@callback( + Output(component_id="acheteur_data", component_property="data"), + Input(component_id="url", component_property="pathname"), +) +def get_acheteur_marches_data(url) -> pl.LazyFrame: + acheteur_siret = url.split("/")[-1] + lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + lff = lff.select( + "uid", + "objet", + "dateNotification", + "titulaire_nom", + "montant", + "codeCPV", + "dureeMois", + ) + lff = lff.sort("dateNotification", descending=True, nulls_last=True) + data = lff.collect(engine="streaming").to_dicts() + return data + + +@callback( + Output(component_id="acheteur_last_marches", component_property="children"), + Input(component_id="acheteur_data", component_property="data"), +) +def get_last_marches_table(data) -> html.Div: + table = html.Div( + className="marches_table", + children=dash_table.DataTable( + data=data[:20], + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "300px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "titulaire_nom"}, + "minWidth": "200px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + ), + ) + return table diff --git a/src/pages/home.py b/src/pages/home.py index 31c1ca4..780bfad 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -47,49 +47,52 @@ schema = lf.collect_schema() title = "Tableau" register_page(__name__, path="/", title="decp.info", name=title, order=1) -datatable = dash_table.DataTable( - cell_selectable=False, - id="table", - page_size=20, - page_current=0, - page_action="custom", - filter_action="custom", - filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": True, - } - for i in lf.collect_schema().names() - ], - sort_action="custom", - sort_mode="multi", - sort_by=[], - row_deletable=False, - style_cell_conditional=[ - { - "if": {"column_id": "objet"}, - "minWidth": "350px", - "textAlign": "left", - "overflow": "hidden", - "lineHeight": "14px", - "whiteSpace": "normal", - }, - { - "if": {"column_id": "acheteur_nom"}, - "minWidth": "250px", - "textAlign": "left", - "overflow": "hidden", - "lineHeight": "14px", - "whiteSpace": "normal", - }, - ], - data_timestamp=0, - markdown_options={"html": True}, +datatable = html.Div( + className="marches_table", + children=dash_table.DataTable( + cell_selectable=False, + id="table", + page_size=20, + page_current=0, + page_action="custom", + filter_action="custom", + filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, + columns=[ + { + "name": i, + "id": i, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": True, + } + for i in lf.collect_schema().names() + ], + sort_action="custom", + sort_mode="multi", + sort_by=[], + row_deletable=False, + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "350px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "acheteur_nom"}, + "minWidth": "250px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + data_timestamp=0, + markdown_options={"html": True}, + ), ) layout = [ From 59321fc80ac66eef25484aed3f644e6c8b205343 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 14:29:48 +0200 Subject: [PATCH 04/41] Commune acheteur, carte localisation #28 --- src/assets/style.css | 17 ++++++++++++++++ src/figures.py | 47 ++++++++++++++++++++++++++++++++++++++----- src/pages/acheteur.py | 43 ++++++++++++++++++++++++++++++++++----- src/utils.py | 4 ++-- 4 files changed, 99 insertions(+), 12 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 88f1e3f..2058973 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -111,3 +111,20 @@ h3 { #_pages_content { padding-top: 28px; } + +/* Vue acheteur/fournisseur */ +.wrapper { + display: grid; + grid-gap: 10px; + margin-bottom: 50px; +} + +.org_infos { + grid-column: 1; + grid-row: 1; +} + +.org_map { + grid-column: 2; + grid-row: 1; +} diff --git a/src/figures.py b/src/figures.py index 8ca0c5d..9be7fa6 100644 --- a/src/figures.py +++ b/src/figures.py @@ -2,7 +2,7 @@ import json import plotly.express as px import polars as pl -from dash import dash_table, html +from dash import dash_table, dcc, html def get_map_count_marches(lf: pl.LazyFrame): @@ -32,11 +32,11 @@ def get_map_count_marches(lf: pl.LazyFrame): df, geojson=departements, locations="Département", - color="uid", - color_continuous_scale="Reds", + # color="uid", + # color_continuous_scale="Reds", title="Nombres de marchés attribués par département (lieu d'exécution)", - range_color=(df["uid"].min(), df["uid"].max()), - labels={"uid": "Marchés attribués"}, + # range_color=(df["uid"].min(), df["uid"].max()), + # labels={"uid": "Marchés attribués"}, scope="europe", width=1000, height=800, @@ -157,3 +157,40 @@ def get_sources_tables(source_path) -> html.Div: datatable.data = df.to_dicts() return html.Div(children=datatable) + + +def point_on_map(lat, lon): + lat = float(lat) + lon = float(lon) + + # Create a scatter mapbox or choropleth map + fig = px.scatter_map( + lat=[lat], + lon=[lon], + height=300, + width=400, + color=[1], + ) + + fig.update_coloraxes(showscale=False) + + # Set map style (you can use 'open-street-map', 'carto-positron', etc.) + fig.update_layout( + mapbox_style="light", # Light, clean background + margin={"r": 0, "t": 0, "l": 0, "b": 0}, + ) + + # Optionally, center the map on France + fig.update_geos( + center=dict(lat=46.603354, lon=1.888334), # Center of France + lataxis_range=[41, 51.5], # Latitude range for France + lonaxis_range=[-5, 10], # Longitude range for France + ) + + # But scatter_mapbox doesn't use geos, so better to control via zoom/center manually + # Let's reset and use proper centering in scatter_mapbox instead: + + fig.update_layout(map_center={"lat": 46.6, "lon": 1.89}, map_zoom=4) + + graph = dcc.Graph(id="map", figure=fig) + return graph diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index f767e6b..5b14664 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,7 +1,8 @@ import polars as pl from dash import Input, Output, callback, dash_table, dcc, html, register_page -from src.utils import lf +from src.figures import point_on_map +from src.utils import get_annuaire_data, lf register_page( __name__, @@ -19,7 +20,27 @@ layout = [ html.Div( className="container", children=[ - html.H2(id="acheteur_title", children=""), + html.H2( + children=[ + html.Span(id="acheteur_siret"), + " - ", + html.Span(id="acheteur_nom"), + ] + ), + html.Div( + className="wrapper", + children=[ + html.Div( + className="org_infos", + children=[html.P(["Commune : ", html.Span(id="commune")])], + ), + html.Div(className="org_map", id="acheteur_map"), + # adresse + # code commune + # lat long + ], + ), + # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics notifiés"), html.Div(id="acheteur_last_marches", children=""), ], @@ -28,15 +49,27 @@ layout = [ @callback( - Output(component_id="acheteur_title", component_property="children"), + Output(component_id="acheteur_siret", component_property="children"), + Output(component_id="acheteur_nom", component_property="children"), + Output(component_id="commune", component_property="children"), + Output(component_id="acheteur_map", component_property="children"), Input(component_id="url", component_property="pathname"), ) def update_acheteur(url): acheteur_siret = url.split("/")[-1] if len(acheteur_siret) != 14: return f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" - - return acheteur_siret + data = get_annuaire_data(acheteur_siret) + data_etablissement = data["matching_etablissements"][0] + acheteur_map = point_on_map( + data_etablissement["latitude"], data_etablissement["longitude"] + ) + return ( + acheteur_siret, + data["nom_raison_sociale"], + data_etablissement["libelle_commune"], + acheteur_map, + ) @callback( diff --git a/src/utils.py b/src/utils.py index 3286b15..cc49a09 100644 --- a/src/utils.py +++ b/src/utils.py @@ -102,10 +102,10 @@ def format_number(number) -> str: return number -def get_acheteur_data(siret: str) -> dict: +def get_annuaire_data(siret: str) -> dict: url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" response = get(url) - return response.json() + return response.json()["results"][0] def get_decp_data() -> pl.LazyFrame: From b88d1c9f9382a913e8ff2469f91482f9ae8213f9 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 18:16:53 +0200 Subject: [PATCH 05/41] =?UTF-8?q?Pagination=20derniers=20march=C3=A9s=20#2?= =?UTF-8?q?8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 5b14664..7db212b 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -101,7 +101,9 @@ def get_last_marches_table(data) -> html.Div: table = html.Div( className="marches_table", children=dash_table.DataTable( - data=data[:20], + data=data, + page_action="native", + page_size=10, style_cell_conditional=[ { "if": {"column_id": "objet"}, From 2fe92d8a1475c1c8103078bda1ec83f88775ba3d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 20:49:15 +0200 Subject: [PATCH 06/41] Plus d'informations, stats #28 --- src/assets/style.css | 27 ++++++++- src/pages/acheteur.py | 129 ++++++++++++++++++++++++++++++++++++------ src/utils.py | 21 ++++++- 3 files changed, 155 insertions(+), 22 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 2058973..c8449d4 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -76,6 +76,7 @@ td[data-dash-column="objet"] { } .marches_table .cell-table tr:nth-child(even) td { background-color: #feeeee; + font-family: sans-serif; } #header > *, @@ -117,14 +118,34 @@ h3 { display: grid; grid-gap: 10px; margin-bottom: 50px; + justify-content: space-between; +} + +.org_title { + grid-column: 1 / 3; + grid-row: 1; +} + +.org_year { + grid-column: 3; + grid-row: 1; } .org_infos { grid-column: 1; - grid-row: 1; + grid-row: 2; +} + +.org_infos > p { + margin: 8px 0; +} + +.org_stats { + grid-column: 2; + grid-row: 2; } .org_map { - grid-column: 2; - grid-row: 1; + grid-column: 3; + grid-row: 2; } diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 7db212b..56a50a9 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,8 +1,10 @@ +import datetime + import polars as pl from dash import Input, Output, callback, dash_table, dcc, html, register_page from src.figures import point_on_map -from src.utils import get_annuaire_data, lf +from src.utils import format_number, get_annuaire_data, get_departement_region, lf register_page( __name__, @@ -20,24 +22,58 @@ layout = [ html.Div( className="container", children=[ - html.H2( - children=[ - html.Span(id="acheteur_siret"), - " - ", - html.Span(id="acheteur_nom"), - ] - ), html.Div( className="wrapper", children=[ + html.H2( + className="org_title", + children=[ + html.Span(id="acheteur_siret"), + " - ", + html.Span(id="acheteur_nom"), + ], + ), + html.Div( + className="org_year", + children=dcc.Dropdown( + id="acheteur_year", + options=["Toutes"] + + [ + str(year) + for year in range( + 2018, int(datetime.date.today().year) + 1 + ) + ], + placeholder="Année", + ), + ), html.Div( className="org_infos", - children=[html.P(["Commune : ", html.Span(id="commune")])], + children=[ + html.P(["Commune : ", html.Strong(id="acheteur_commune")]), + html.P( + [ + "Département : ", + html.Strong(id="acheteur_departement"), + ] + ), + html.P(["Région : ", html.Strong(id="acheteur_region")]), + html.A( + id="acheteur_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + target="_blank", + ), + ], + ), + html.Div( + className="org_stats", + children=[ + html.P(id="acheteur_titre_stats"), + html.P(id="acheteur_marches_notifies"), + html.P(id="acheteur_fournisseurs_differents"), + ], ), html.Div(className="org_map", id="acheteur_map"), - # adresse - # code commune - # lat long ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -51,38 +87,85 @@ layout = [ @callback( Output(component_id="acheteur_siret", component_property="children"), Output(component_id="acheteur_nom", component_property="children"), - Output(component_id="commune", component_property="children"), + Output(component_id="acheteur_commune", component_property="children"), Output(component_id="acheteur_map", component_property="children"), + Output(component_id="acheteur_departement", component_property="children"), + Output(component_id="acheteur_region", component_property="children"), + Output(component_id="acheteur_lien_annuaire", component_property="href"), Input(component_id="url", component_property="pathname"), ) -def update_acheteur(url): +def update_acheteur_infos(url): acheteur_siret = url.split("/")[-1] if len(acheteur_siret) != 14: - return f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" + acheteur_siret = ( + f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" + ) data = get_annuaire_data(acheteur_siret) data_etablissement = data["matching_etablissements"][0] acheteur_map = point_on_map( data_etablissement["latitude"], data_etablissement["longitude"] ) + code_departement, nom_departement, nom_region = get_departement_region( + data_etablissement["code_postal"] + ) + departement = f"{nom_departement} ({code_departement})" + lien_annuaire = ( + f"https://annuaire-entreprises.data.gouv.fr/etablissement/{acheteur_siret}" + ) return ( acheteur_siret, data["nom_raison_sociale"], data_etablissement["libelle_commune"], acheteur_map, + departement, + nom_region, + lien_annuaire, ) +@callback( + Output(component_id="acheteur_marches_notifies", component_property="children"), + Output( + component_id="acheteur_fournisseurs_differents", component_property="children" + ), + Input(component_id="acheteur_data", component_property="data"), +) +def update_acheteur_stats(data): + df = pl.DataFrame(data) + df_marches = df.unique("uid") + nb_marches = format_number(df_marches.height) + # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) + marches_notifies = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] + # + ", pour un total de ", html.Strong(somme_marches + " €")] + del df_marches + + nb_fournisseurs = df.unique("titulaire_id").height + nb_fournisseurs = [ + html.Strong(format_number(nb_fournisseurs)), + " fournisseurs (SIRET) différents", + ] + del df + + return marches_notifies, nb_fournisseurs + + @callback( Output(component_id="acheteur_data", component_property="data"), Input(component_id="url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url) -> pl.LazyFrame: +def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: acheteur_siret = url.split("/")[-1] lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + if acheteur_year and acheteur_year != "Toutes": + lff = lff.filter( + pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) + ) lff = lff.select( "uid", "objet", "dateNotification", + "titulaire_id", "titulaire_nom", "montant", "codeCPV", @@ -98,11 +181,25 @@ def get_acheteur_marches_data(url) -> pl.LazyFrame: Input(component_id="acheteur_data", component_property="data"), ) def get_last_marches_table(data) -> html.Div: + columns = data[0].keys() + table = html.Div( className="marches_table", children=dash_table.DataTable( data=data, page_action="native", + columns=[ + { + "name": i, + "id": i, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": False, + } + for i in columns + if i not in ["titulaire_id"] + ], page_size=10, style_cell_conditional=[ { diff --git a/src/utils.py b/src/utils.py index cc49a09..9900ee9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,3 +1,4 @@ +import json import logging import os from time import sleep @@ -124,9 +125,6 @@ def get_decp_data() -> pl.LazyFrame: sleep(10) lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH")) - # Remplacement des valeurs numériques par des chaînes de caractères - # lff = numbers_to_strings(lff) - # Tri des marchés par date de notification lff = lff.sort(by=["dateNotification"], descending=True, nulls_last=True) @@ -136,4 +134,21 @@ def get_decp_data() -> pl.LazyFrame: return lff +def get_departements() -> dict: + with open("data/departements.json", "rb") as f: + data = json.load(f) + return data + + +def get_departement_region(code_postal): + if code_postal > "97000": + code_departement = code_postal[:3] + else: + code_departement = code_postal[:2] + nom_departement = departements[code_departement]["departement"] + nom_region = departements[code_departement]["region"] + return code_departement, nom_departement, nom_region + + lf = get_decp_data() +departements = get_departements() From 9e5673c28f8418adde4187f9e377c9b7fef2ec0a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 21:14:11 +0200 Subject: [PATCH 07/41] Ajout des liens dans le tableau le plus tard possible --- src/pages/home.py | 48 ++++++++++++++++++++++------------------------- src/utils.py | 16 ++++++++-------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index 780bfad..971fff3 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -34,13 +34,6 @@ lf = booleans_to_strings(lf) # Remplacer les valeurs manquantes par des chaînes vides lf = lf.fill_null("") - -# Ajout des liens vers l'annuaire -lf = add_annuaire_link(lf) - -# Ajout des liens open data -lf = add_resource_link(lf) - schema = lf.collect_schema() @@ -57,17 +50,6 @@ datatable = html.Div( page_action="custom", filter_action="custom", filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": True, - } - for i in lf.collect_schema().names() - ], sort_action="custom", sort_mode="multi", sort_by=[], @@ -158,6 +140,7 @@ layout = [ @callback( Output("table", "data"), + Output("table", "columns"), Output("table", "data_timestamp"), Output("nb_rows", "children"), Output("btn-download-data", "disabled"), @@ -229,6 +212,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) start_row = page_current * page_size # end_row = (page_current + 1) * page_size dff = dff.slice(start_row, page_size) + + # Ajout des liens vers l'annuaire des entreprises + dff = add_annuaire_link(dff) + + # Ajout des liens vers les fichiers Open Data + dff = add_resource_link(dff) + + # Liste finale de colonnes + columns = [ + { + "name": column, + "id": column, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": True, + } + for column in dff.columns + ] + dicts = dff.to_dicts() if height > 65000: @@ -242,6 +245,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) return ( dicts, + columns, data_timestamp + 1, nb_rows, download_disabled, @@ -259,14 +263,6 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) def download_data(n_clicks, hidden_columns: list = None): df_to_download = df_filtered.clone() - print(df_to_download.columns) - - # Rétablissement des colonnes source et sourceOpenData (voir add_resource_link) - df_to_download = df_to_download.with_columns( - pl.col("source").str.extract(r'href="(.*?)"').alias("sourceFile"), - pl.col("source").str.extract(r'">(.*?)<').alias("sourceDataset"), - ) - # Les colonnes masquées sont supprimées if hidden_columns: df_to_download = df_to_download.drop(hidden_columns) diff --git a/src/utils.py b/src/utils.py index 9900ee9..a7ef63c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -42,18 +42,18 @@ def split_filter_part(filter_part): return [None] * 3 -def add_resource_link(lff: pl.LazyFrame) -> pl.LazyFrame: - lff = lff.with_columns( +def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame: + dff = dff.with_columns( ( '' + pl.col("sourceDataset") + "" ).alias("source") ) - lff = lff.drop(["sourceFile", "sourceDataset"]) - return lff + dff = dff.drop(["sourceFile", "sourceDataset"]) + return dff -def add_annuaire_link(lff: pl.LazyFrame): - lff = lff.with_columns( +def add_annuaire_link(dff: pl.DataFrame): + dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( '" ).alias("acheteur_id") ) - return lff + return dff def booleans_to_strings(lff: pl.LazyFrame) -> pl.LazyFrame: From e88992a470eb579138a800c24d5d78817f15258a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 21:39:06 +0200 Subject: [PATCH 08/41] =?UTF-8?q?Le=20lien=20acheteur=20m=C3=A8ne=20mainte?= =?UTF-8?q?nant=20=C3=A0=20la=20page=20acheteur=20au=20lieu=20de=20l'annua?= =?UTF-8?q?ire=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app.py | 1 + src/figures.py | 6 +----- src/pages/home.py | 4 ++-- src/utils.py | 6 +++--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/app.py b/src/app.py index e45d9ef..66e5f30 100644 --- a/src/app.py +++ b/src/app.py @@ -74,6 +74,7 @@ app.layout = html.Div( page["name"], href=page["relative_path"], className="nav" ) for page in page_registry.values() + if page["name"] not in ["Acheteur", "Fournisseur"] ] ), ], diff --git a/src/figures.py b/src/figures.py index 9be7fa6..3361e06 100644 --- a/src/figures.py +++ b/src/figures.py @@ -165,11 +165,7 @@ def point_on_map(lat, lon): # Create a scatter mapbox or choropleth map fig = px.scatter_map( - lat=[lat], - lon=[lon], - height=300, - width=400, - color=[1], + lat=[lat], lon=[lon], height=300, width=400, color=[1], size=[1] ) fig.update_coloraxes(showscale=False) diff --git a/src/pages/home.py b/src/pages/home.py index 971fff3..3377440 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -6,7 +6,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from dotenv import load_dotenv from src.utils import ( - add_annuaire_link, + add_org_links, add_resource_link, booleans_to_strings, format_number, @@ -214,7 +214,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) dff = dff.slice(start_row, page_size) # Ajout des liens vers l'annuaire des entreprises - dff = add_annuaire_link(dff) + dff = add_org_links(dff) # Ajout des liens vers les fichiers Open Data dff = add_resource_link(dff) diff --git a/src/utils.py b/src/utils.py index a7ef63c..f7df698 100644 --- a/src/utils.py +++ b/src/utils.py @@ -52,7 +52,7 @@ def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame: return dff -def add_annuaire_link(dff: pl.DataFrame): +def add_org_links(dff: pl.DataFrame): dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( @@ -67,9 +67,9 @@ def add_annuaire_link(dff: pl.DataFrame): ) dff = dff.with_columns( ( - '' + + '" target="_blank">' + pl.col("acheteur_id") + "" ).alias("acheteur_id") From 8f335a52ea91a59d69db09bebea9be24f81df8e5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 21:50:33 +0200 Subject: [PATCH 09/41] =?UTF-8?q?Notifi=C3=A9=20=3D>=20attribu=C3=A9=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 56a50a9..f47eca6 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -69,7 +69,7 @@ layout = [ className="org_stats", children=[ html.P(id="acheteur_titre_stats"), - html.P(id="acheteur_marches_notifies"), + html.P(id="acheteur_marches_attribues"), html.P(id="acheteur_fournisseurs_differents"), ], ), @@ -77,7 +77,7 @@ layout = [ ], ), # récupérer les données de l'acheteur sur l'api annuaire - html.H3("Derniers marchés publics notifiés"), + html.H3("Derniers marchés publics attribués"), html.Div(id="acheteur_last_marches", children=""), ], ), @@ -124,7 +124,7 @@ def update_acheteur_infos(url): @callback( - Output(component_id="acheteur_marches_notifies", component_property="children"), + Output(component_id="acheteur_marches_attribues", component_property="children"), Output( component_id="acheteur_fournisseurs_differents", component_property="children" ), @@ -135,7 +135,7 @@ def update_acheteur_stats(data): df_marches = df.unique("uid") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) - marches_notifies = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] + marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] # + ", pour un total de ", html.Strong(somme_marches + " €")] del df_marches @@ -146,7 +146,7 @@ def update_acheteur_stats(data): ] del df - return marches_notifies, nb_fournisseurs + return marches_attribues, nb_fournisseurs @callback( From 944f207d9b1b949427f64db5cc381bbef585ea50 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 23:15:20 +0200 Subject: [PATCH 10/41] =?UTF-8?q?T=C3=A9l=C3=A9chargement=20des=20donn?= =?UTF-8?q?=C3=A9es=20acheteur=20filtr=C3=A9es=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 60 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index f47eca6..ded95ea 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -71,6 +71,12 @@ layout = [ html.P(id="acheteur_titre_stats"), html.P(id="acheteur_marches_attribues"), html.P(id="acheteur_fournisseurs_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-acheteur-data", + ), + dcc.Download(id="download-acheteur-data"), + dcc.Store(id="n_acheteur_downloads"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -132,6 +138,8 @@ def update_acheteur_infos(url): ) def update_acheteur_stats(data): df = pl.DataFrame(data) + if df.height == 0: + df = pl.DataFrame(schema=lf.collect_schema()) df_marches = df.unique("uid") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) @@ -157,10 +165,6 @@ def update_acheteur_stats(data): def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: acheteur_siret = url.split("/")[-1] lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) - if acheteur_year and acheteur_year != "Toutes": - lff = lff.filter( - pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) - ) lff = lff.select( "uid", "objet", @@ -171,7 +175,12 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: "codeCPV", "dureeMois", ) + if acheteur_year and acheteur_year != "Toutes": + lff = lff.filter( + pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) + ) lff = lff.sort("dateNotification", descending=True, nulls_last=True) + data = lff.collect(engine="streaming").to_dicts() return data @@ -181,10 +190,19 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: Input(component_id="acheteur_data", component_property="data"), ) def get_last_marches_table(data) -> html.Div: - columns = data[0].keys() + columns = [ + "uid", + "objet", + "dateNotification", + "titulaire_nom", + "montant", + "codeCPV", + "dureeMois", + ] table = html.Div( className="marches_table", + id="marches_datatable", children=dash_table.DataTable( data=data, page_action="native", @@ -222,3 +240,35 @@ def get_last_marches_table(data) -> html.Div: ), ) return table + + +@callback( + Output("download-acheteur-data", "data"), + Output("n_acheteur_downloads", "data"), + Input("btn-download-acheteur-data", "n_clicks"), + Input(component_id="acheteur_data", component_property="data"), + Input(component_id="acheteur_nom", component_property="children"), + Input(component_id="acheteur_year", component_property="value"), + Input("n_acheteur_downloads", "data"), + prevent_initial_call=True, +) +def download_acheteur_data( + n_clicks, data: [dict], acheteur_nom: str, annee: str, n_downloads: int +): + if n_clicks is None: + return None, 0 + if n_clicks == n_downloads: + return None, n_downloads + + df_to_download = pl.DataFrame(data) + + def to_bytes(buffer): + df_to_download.write_excel( + buffer, worksheet="DECP" if annee in ["Toutes", None] else annee + ) + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + n_downloads += 1 + return dcc.send_bytes( + to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx" + ), n_downloads From bfa181cefd9de01dca6b1c3632eb9dfa5c5a251b Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 23:57:51 +0200 Subject: [PATCH 11/41] Suppression de la variable globale df_filtered, et adaptations --- src/pages/home.py | 70 ++++++++++++++++------------------------------- src/utils.py | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index 3377440..81ad241 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -9,17 +9,16 @@ from src.utils import ( add_org_links, add_resource_link, booleans_to_strings, + filter_table_data, format_number, lf, - logger, - split_filter_part, + sort_table_data, ) load_dotenv() update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -df_filtered = pl.DataFrame() # Suppression des colonnes inutiles lf = lf.drop( @@ -128,6 +127,7 @@ layout = [ disabled=True, ), dcc.Download(id="download-data"), + dcc.Store(id="filtered_data", storage_type="memory"), html.P("Données mises à jour le " + str(update_date)), ], className="table-menu", @@ -141,6 +141,7 @@ layout = [ @callback( Output("table", "data"), Output("table", "columns"), + # Output("filtered_data", "data"), Output("table", "data_timestamp"), Output("nb_rows", "children"), Output("btn-download-data", "disabled"), @@ -154,55 +155,22 @@ layout = [ ) def update_table(page_current, page_size, filter_query, sort_by, data_timestamp): print(" + + + + + + + + + + + + + + + + + + ") - global df_filtered # Application des filtres lff: pl.LazyFrame = lf # start from the original data if filter_query: - filtering_expressions = filter_query.split(" && ") - for filter_part in filtering_expressions: - col_name, operator, filter_value = split_filter_part(filter_part) - col_type = str(schema[col_name]) - print("filter_value:", filter_value) - print("filter_value_type:", type(filter_value)) - print("col_type:", col_type) - - if operator in ("<", "<=", ">", ">="): - filter_value = int(filter_value) - if operator == "<": - lff = lff.filter(pl.col(col_name) < filter_value) - elif operator == ">": - lff = lff.filter(pl.col(col_name) > filter_value) - elif operator == ">=": - lff = lff.filter(pl.col(col_name) >= filter_value) - elif operator == "<=": - lff = lff.filter(pl.col(col_name) <= filter_value) - - elif col_type.startswith("Int") or col_type.startswith("Float"): - try: - filter_value = int(filter_value) - except ValueError: - logger.error(f"Invalid numeric filter value: {filter_value}") - continue - lff = lff.filter(pl.col(col_name) == filter_value) - - elif operator == "contains" and col_type == "String": - lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) - - # elif operator == 'datestartswith': - # lff = lff.filter(pl.col(col_name).str.startswith(filter_value)") + lff = filter_table_data(lff, filter_query) if len(sort_by) > 0: - lff = lff.sort( - [col["column_id"] for col in sort_by], - descending=[col["direction"] == "desc" for col in sort_by], - nulls_last=True, - ) - print(sort_by) + lff = sort_table_data(lff, sort_by) + # Matérialisation des filtres dff: pl.DataFrame = lff.collect() - df_filtered = dff.clone() + # if filter_query or sort_by: + # filtered_data = dff.to_dicts() + # else: + # filtered_data = {} height = dff.height @@ -257,18 +225,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) @callback( Output("download-data", "data"), Input("btn-download-data", "n_clicks"), + State("table", "filter_query"), + State("table", "sort_by"), State("table", "hidden_columns"), prevent_initial_call=True, ) -def download_data(n_clicks, hidden_columns: list = None): - df_to_download = df_filtered.clone() +def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None): + lff: pl.LazyFrame = lf # start from the original data # Les colonnes masquées sont supprimées if hidden_columns: - df_to_download = df_to_download.drop(hidden_columns) + lff = lff.drop(hidden_columns) + + if filter_query: + lff = filter_table_data(lff, filter_query) + + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) def to_bytes(buffer): - df_to_download.write_excel(buffer, worksheet="DECP") + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return dcc.send_bytes(to_bytes, filename=f"decp_{date}.xlsx") diff --git a/src/utils.py b/src/utils.py index f7df698..1cd916c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -150,5 +150,53 @@ def get_departement_region(code_postal): return code_departement, nom_departement, nom_region +def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: + schema = lff.collect_schema() + filtering_expressions = filter_query.split(" && ") + for filter_part in filtering_expressions: + col_name, operator, filter_value = split_filter_part(filter_part) + col_type = str(schema[col_name]) + print("filter_value:", filter_value) + print("filter_value_type:", type(filter_value)) + print("col_type:", col_type) + + if operator in ("<", "<=", ">", ">="): + filter_value = int(filter_value) + if operator == "<": + lff = lff.filter(pl.col(col_name) < filter_value) + elif operator == ">": + lff = lff.filter(pl.col(col_name) > filter_value) + elif operator == ">=": + lff = lff.filter(pl.col(col_name) >= filter_value) + elif operator == "<=": + lff = lff.filter(pl.col(col_name) <= filter_value) + + elif col_type.startswith("Int") or col_type.startswith("Float"): + try: + filter_value = int(filter_value) + except ValueError: + logger.error(f"Invalid numeric filter value: {filter_value}") + continue + lff = lff.filter(pl.col(col_name) == filter_value) + + elif operator == "contains" and col_type == "String": + lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) + + # elif operator == 'datestartswith': + # lff = lff.filter(pl.col(col_name).str.startswith(filter_value)") + + return lff + + +def sort_table_data(lff: pl.LazyFrame, sort_by: list) -> pl.LazyFrame: + lff = lff.sort( + [col["column_id"] for col in sort_by], + descending=[col["direction"] == "desc" for col in sort_by], + nulls_last=True, + ) + print(sort_by) + return lff + + lf = get_decp_data() departements = get_departements() From 1c5868d75fd5f50509f7d18540cf109d4e032dec Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 27 Sep 2025 23:58:23 +0200 Subject: [PATCH 12/41] =?UTF-8?q?Utilisation=20de=20State=20plut=C3=B4t=20?= =?UTF-8?q?qu'Input=20pour=20ne=20pas=20trigger=20le=20download=20#28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index ded95ea..965ccdd 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,7 +1,7 @@ import datetime import polars as pl -from dash import Input, Output, callback, dash_table, dcc, html, register_page +from dash import Input, Output, State, callback, dash_table, dcc, html, register_page from src.figures import point_on_map from src.utils import format_number, get_annuaire_data, get_departement_region, lf @@ -50,6 +50,7 @@ layout = [ html.Div( className="org_infos", children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. html.P(["Commune : ", html.Strong(id="acheteur_commune")]), html.P( [ @@ -76,7 +77,6 @@ layout = [ id="btn-download-acheteur-data", ), dcc.Download(id="download-acheteur-data"), - dcc.Store(id="n_acheteur_downloads"), ], ), html.Div(className="org_map", id="acheteur_map"), @@ -244,22 +244,18 @@ def get_last_marches_table(data) -> html.Div: @callback( Output("download-acheteur-data", "data"), - Output("n_acheteur_downloads", "data"), Input("btn-download-acheteur-data", "n_clicks"), - Input(component_id="acheteur_data", component_property="data"), - Input(component_id="acheteur_nom", component_property="children"), - Input(component_id="acheteur_year", component_property="value"), - Input("n_acheteur_downloads", "data"), + State(component_id="acheteur_data", component_property="data"), + State(component_id="acheteur_nom", component_property="children"), + State(component_id="acheteur_year", component_property="value"), prevent_initial_call=True, ) def download_acheteur_data( - n_clicks, data: [dict], acheteur_nom: str, annee: str, n_downloads: int + n_clicks, + data: [dict], + acheteur_nom: str, + annee: str, ): - if n_clicks is None: - return None, 0 - if n_clicks == n_downloads: - return None, n_downloads - df_to_download = pl.DataFrame(data) def to_bytes(buffer): @@ -268,7 +264,4 @@ def download_acheteur_data( ) date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") - n_downloads += 1 - return dcc.send_bytes( - to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx" - ), n_downloads + return dcc.send_bytes(to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx") From 98b8699afbc7ed5989bdc172f4e9f874dd78e3af Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 00:30:52 +0200 Subject: [PATCH 13/41] =?UTF-8?q?R=C3=A9tablissement=20de=20la=20carte=20d?= =?UTF-8?q?es=20march=C3=A9s=20par=20dpt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/style.css | 5 +++++ src/figures.py | 8 ++++---- src/pages/acheteur.py | 9 +++++---- src/pages/home.py | 22 +++------------------- src/utils.py | 11 +++++++++-- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index c8449d4..873998a 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -106,6 +106,11 @@ a.nav { } h3 { + margin: 36px 0 24px 0; +} + +summary > h3 { + margin: 0; display: inline; } diff --git a/src/figures.py b/src/figures.py index 3361e06..860c695 100644 --- a/src/figures.py +++ b/src/figures.py @@ -32,11 +32,11 @@ def get_map_count_marches(lf: pl.LazyFrame): df, geojson=departements, locations="Département", - # color="uid", - # color_continuous_scale="Reds", + color="uid", + color_continuous_scale="Reds", title="Nombres de marchés attribués par département (lieu d'exécution)", - # range_color=(df["uid"].min(), df["uid"].max()), - # labels={"uid": "Marchés attribués"}, + range_color=(df["uid"].min(), df["uid"].max()), + labels={"uid": "Marchés attribués"}, scope="europe", width=1000, height=800, diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 965ccdd..e3fa979 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -140,7 +140,7 @@ def update_acheteur_stats(data): df = pl.DataFrame(data) if df.height == 0: df = pl.DataFrame(schema=lf.collect_schema()) - df_marches = df.unique("uid") + df_marches = df.unique("id") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] @@ -165,8 +165,9 @@ def update_acheteur_stats(data): def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: acheteur_siret = url.split("/")[-1] lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + lff = lff.fill_null("") lff = lff.select( - "uid", + "id", "objet", "dateNotification", "titulaire_id", @@ -179,7 +180,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: lff = lff.filter( pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) ) - lff = lff.sort("dateNotification", descending=True, nulls_last=True) + lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) data = lff.collect(engine="streaming").to_dicts() return data @@ -191,7 +192,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: ) def get_last_marches_table(data) -> html.Div: columns = [ - "uid", + "id", "objet", "dateNotification", "titulaire_nom", diff --git a/src/pages/home.py b/src/pages/home.py index 81ad241..9e19d8c 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -8,7 +8,6 @@ from dotenv import load_dotenv from src.utils import ( add_org_links, add_resource_link, - booleans_to_strings, filter_table_data, format_number, lf, @@ -20,19 +19,6 @@ load_dotenv() update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -# Suppression des colonnes inutiles -lf = lf.drop( - [ - "donneesActuelles", - ] -) - -# Convertir les colonnes booléennes en chaînes de caractères -lf = booleans_to_strings(lf) - -# Remplacer les valeurs manquantes par des chaînes vides -lf = lf.fill_null("") - schema = lf.collect_schema() @@ -164,14 +150,12 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) if len(sort_by) > 0: lff = sort_table_data(lff, sort_by) + # Remplace les strings null par "", mais pas les numeric null + lff = lff.fill_null("") + # Matérialisation des filtres dff: pl.DataFrame = lff.collect() - # if filter_query or sort_by: - # filtered_data = dff.to_dicts() - # else: - # filtered_data = {} - height = dff.height nb_rows = f"{format_number(height)} lignes" diff --git a/src/utils.py b/src/utils.py index 1cd916c..96c2bfb 100644 --- a/src/utils.py +++ b/src/utils.py @@ -126,10 +126,17 @@ def get_decp_data() -> pl.LazyFrame: lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH")) # Tri des marchés par date de notification - lff = lff.sort(by=["dateNotification"], descending=True, nulls_last=True) + lff = lff.sort(by=["dateNotification", "uid"], descending=True, nulls_last=True) # Uniquement les données actuelles, pas les anciennes versions de marchés - lff = lff.filter(pl.col("donneesActuelles")) + lff = lff.filter(pl.col("donneesActuelles")).drop("donneesActuelles") + + # Convertir les colonnes booléennes en chaînes de caractères + lff = booleans_to_strings(lff) + + # Bizarrement je ne peux pas faire lff = lff.fill_null("") ici + # ça génère une erreur dans la page acheteur (acheteur_data.table) : + # AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import) return lff From 4b8565dcf2e11752a90acb6eb48db40ea772ec92 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 00:33:21 +0200 Subject: [PATCH 14/41] Chanlog --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7092317..70a2fae 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,12 @@ python run.py ## Notes de version -#### 2.0.1 (23 septembre 2025) +#### 2.1.0 + +- Ajout de la vue acheteur +- Variables globales uniquement en lecture + +##### 2.0.1 (23 septembre 2025) - Bloquage du bouton de téléchargement si trop de lignes (+ 65000) [#38](https://github.com/ColinMaudry/decp.info/issues/38) - Amélioration du script de déploiement (deploy.sh) From 0e20b0012f1acf9bf0257bdb90c77e05589c771a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 00:34:47 +0200 Subject: [PATCH 15/41] =?UTF-8?q?D=C3=A9but=20de=20vue=20fournisseur=20#35?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/fournisseur.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/pages/fournisseur.py diff --git a/src/pages/fournisseur.py b/src/pages/fournisseur.py new file mode 100644 index 0000000..509ca1f --- /dev/null +++ b/src/pages/fournisseur.py @@ -0,0 +1,13 @@ +from dash import html, register_page + +register_page( + __name__, + path_template="/fournisseur/", + title="decp.info - fournisseur", + name="Fournisseur", + order=5, +) + +layout = [ + html.Div(className="container", children=["Cette page est encore en construction."]) +] From 8d236ab8dcd90807326937d3ff0503bcd7d6c07a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 17:57:32 +0200 Subject: [PATCH 16/41] Module httpx --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5d086ef..74be8fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "python-dotenv", "xlsxwriter", "plotly[express]", + "httpx" ] [project.optional-dependencies] From cc05ac0d99b706d35b4f51e512cd747465e12e2b Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 18:02:31 +0200 Subject: [PATCH 17/41] departements.json --- data/departements.json | 398 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 data/departements.json diff --git a/data/departements.json b/data/departements.json new file mode 100644 index 0000000..4115895 --- /dev/null +++ b/data/departements.json @@ -0,0 +1,398 @@ +{ + "01": { + "departement": "Ain", + "region": "Auvergne-Rhône-Alpes" + }, + "02": { + "departement": "Aisne", + "region": "Hauts-de-France" + }, + "03": { + "departement": "Allier", + "region": "Auvergne-Rhône-Alpes" + }, + "04": { + "departement": "Alpes-de-Haute-Provence", + "region": "Provence-Alpes-Côte d'Azur" + }, + "05": { + "departement": "Hautes-Alpes", + "region": "Provence-Alpes-Côte d'Azur" + }, + "06": { + "departement": "Alpes-Maritimes", + "region": "Provence-Alpes-Côte d'Azur" + }, + "07": { + "departement": "Ardèche", + "region": "Auvergne-Rhône-Alpes" + }, + "08": { + "departement": "Ardennes", + "region": "Grand Est" + }, + "09": { + "departement": "Ariège", + "region": "Occitanie" + }, + "10": { + "departement": "Aube", + "region": "Grand Est" + }, + "11": { + "departement": "Aude", + "region": "Occitanie" + }, + "12": { + "departement": "Aveyron", + "region": "Occitanie" + }, + "13": { + "departement": "Bouches-du-Rhône", + "region": "Provence-Alpes-Côte d'Azur" + }, + "14": { + "departement": "Calvados", + "region": "Normandie" + }, + "15": { + "departement": "Cantal", + "region": "Auvergne-Rhône-Alpes" + }, + "16": { + "departement": "Charente", + "region": "Nouvelle-Aquitaine" + }, + "17": { + "departement": "Charente-Maritime", + "region": "Nouvelle-Aquitaine" + }, + "18": { + "departement": "Cher", + "region": "Centre-Val de Loire" + }, + "19": { + "departement": "Corrèze", + "region": "Nouvelle-Aquitaine" + }, + "21": { + "departement": "Côte-d'Or", + "region": "Bourgogne-Franche-Comté" + }, + "22": { + "departement": "Côtes-d'Armor", + "region": "Bretagne" + }, + "23": { + "departement": "Creuse", + "region": "Nouvelle-Aquitaine" + }, + "24": { + "departement": "Dordogne", + "region": "Nouvelle-Aquitaine" + }, + "25": { + "departement": "Doubs", + "region": "Bourgogne-Franche-Comté" + }, + "26": { + "departement": "Drôme", + "region": "Auvergne-Rhône-Alpes" + }, + "27": { + "departement": "Eure", + "region": "Normandie" + }, + "28": { + "departement": "Eure-et-Loir", + "region": "Centre-Val de Loire" + }, + "29": { + "departement": "Finistère", + "region": "Bretagne" + }, + "30": { + "departement": "Gard", + "region": "Occitanie" + }, + "31": { + "departement": "Haute-Garonne", + "region": "Occitanie" + }, + "32": { + "departement": "Gers", + "region": "Occitanie" + }, + "33": { + "departement": "Gironde", + "region": "Nouvelle-Aquitaine" + }, + "34": { + "departement": "Hérault", + "region": "Occitanie" + }, + "35": { + "departement": "Ille-et-Vilaine", + "region": "Bretagne" + }, + "36": { + "departement": "Indre", + "region": "Centre-Val de Loire" + }, + "37": { + "departement": "Indre-et-Loire", + "region": "Centre-Val de Loire" + }, + "38": { + "departement": "Isère", + "region": "Auvergne-Rhône-Alpes" + }, + "39": { + "departement": "Jura", + "region": "Bourgogne-Franche-Comté" + }, + "40": { + "departement": "Landes", + "region": "Nouvelle-Aquitaine" + }, + "41": { + "departement": "Loir-et-Cher", + "region": "Centre-Val de Loire" + }, + "42": { + "departement": "Loire", + "region": "Auvergne-Rhône-Alpes" + }, + "43": { + "departement": "Haute-Loire", + "region": "Auvergne-Rhône-Alpes" + }, + "44": { + "departement": "Loire-Atlantique", + "region": "Pays de la Loire" + }, + "45": { + "departement": "Loiret", + "region": "Centre-Val de Loire" + }, + "46": { + "departement": "Lot", + "region": "Occitanie" + }, + "47": { + "departement": "Lot-et-Garonne", + "region": "Nouvelle-Aquitaine" + }, + "48": { + "departement": "Lozère", + "region": "Occitanie" + }, + "49": { + "departement": "Maine-et-Loire", + "region": "Pays de la Loire" + }, + "50": { + "departement": "Manche", + "region": "Normandie" + }, + "51": { + "departement": "Marne", + "region": "Grand Est" + }, + "52": { + "departement": "Haute-Marne", + "region": "Grand Est" + }, + "53": { + "departement": "Mayenne", + "region": "Pays de la Loire" + }, + "54": { + "departement": "Meurthe-et-Moselle", + "region": "Grand Est" + }, + "55": { + "departement": "Meuse", + "region": "Grand Est" + }, + "56": { + "departement": "Morbihan", + "region": "Bretagne" + }, + "57": { + "departement": "Moselle", + "region": "Grand Est" + }, + "58": { + "departement": "Nièvre", + "region": "Bourgogne-Franche-Comté" + }, + "59": { + "departement": "Nord", + "region": "Hauts-de-France" + }, + "60": { + "departement": "Oise", + "region": "Hauts-de-France" + }, + "61": { + "departement": "Orne", + "region": "Normandie" + }, + "62": { + "departement": "Pas-de-Calais", + "region": "Hauts-de-France" + }, + "63": { + "departement": "Puy-de-Dôme", + "region": "Auvergne-Rhône-Alpes" + }, + "64": { + "departement": "Pyrénées-Atlantiques", + "region": "Nouvelle-Aquitaine" + }, + "65": { + "departement": "Hautes-Pyrénées", + "region": "Occitanie" + }, + "66": { + "departement": "Pyrénées-Orientales", + "region": "Occitanie" + }, + "67": { + "departement": "Bas-Rhin", + "region": "Grand Est" + }, + "68": { + "departement": "Haut-Rhin", + "region": "Grand Est" + }, + "69": { + "departement": "Rhône", + "region": "Auvergne-Rhône-Alpes" + }, + "70": { + "departement": "Haute-Saône", + "region": "Bourgogne-Franche-Comté" + }, + "71": { + "departement": "Saône-et-Loire", + "region": "Bourgogne-Franche-Comté" + }, + "72": { + "departement": "Sarthe", + "region": "Pays de la Loire" + }, + "73": { + "departement": "Savoie", + "region": "Auvergne-Rhône-Alpes" + }, + "74": { + "departement": "Haute-Savoie", + "region": "Auvergne-Rhône-Alpes" + }, + "75": { + "departement": "Paris", + "region": "Île-de-France" + }, + "76": { + "departement": "Seine-Maritime", + "region": "Normandie" + }, + "77": { + "departement": "Seine-et-Marne", + "region": "Île-de-France" + }, + "78": { + "departement": "Yvelines", + "region": "Île-de-France" + }, + "79": { + "departement": "Deux-Sèvres", + "region": "Nouvelle-Aquitaine" + }, + "80": { + "departement": "Somme", + "region": "Hauts-de-France" + }, + "81": { + "departement": "Tarn", + "region": "Occitanie" + }, + "82": { + "departement": "Tarn-et-Garonne", + "region": "Occitanie" + }, + "83": { + "departement": "Var", + "region": "Provence-Alpes-Côte d'Azur" + }, + "84": { + "departement": "Vaucluse", + "region": "Provence-Alpes-Côte d'Azur" + }, + "85": { + "departement": "Vendée", + "region": "Pays de la Loire" + }, + "86": { + "departement": "Vienne", + "region": "Nouvelle-Aquitaine" + }, + "87": { + "departement": "Haute-Vienne", + "region": "Nouvelle-Aquitaine" + }, + "88": { + "departement": "Vosges", + "region": "Grand Est" + }, + "89": { + "departement": "Yonne", + "region": "Bourgogne-Franche-Comté" + }, + "90": { + "departement": "Territoire de Belfort", + "region": "Bourgogne-Franche-Comté" + }, + "91": { + "departement": "Essonne", + "region": "Île-de-France" + }, + "92": { + "departement": "Hauts-de-Seine", + "region": "Île-de-France" + }, + "93": { + "departement": "Seine-Saint-Denis", + "region": "Île-de-France" + }, + "94": { + "departement": "Val-de-Marne", + "region": "Île-de-France" + }, + "95": { + "departement": "Val-d'Oise", + "region": "Île-de-France" + }, + "971": { + "departement": "Guadeloupe", + "region": "Guadeloupe" + }, + "972": { + "departement": "Martinique", + "region": "Martinique" + }, + "973": { + "departement": "Guyane", + "region": "Guyane" + }, + "974": { + "departement": "La Réunion", + "region": "La Réunion" + }, + "976": { + "departement": "Mayotte", + "region": "Mayotte" + } +} From 8a63f72bee77398ae0f7a5c7d1e639a69ab7db0e Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 18:40:00 +0200 Subject: [PATCH 18/41] Vue fournisseur minimaliste #35 --- pyproject.toml | 3 +- src/pages/acheteur.py | 2 +- src/pages/fournisseur.py | 266 ++++++++++++++++++++++++++++++++++++++- src/utils.py | 4 +- 4 files changed, 268 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 74be8fa..e533afa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,8 @@ dependencies = [ "python-dotenv", "xlsxwriter", "plotly[express]", - "httpx" + "httpx", + "pandas" # utilisé pour la création de certains graphiques ] [project.optional-dependencies] diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index e3fa979..a70ff04 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -8,7 +8,7 @@ from src.utils import format_number, get_annuaire_data, get_departement_region, register_page( __name__, - path_template="/acheteur/", + path_template="/acheteurs/", title="decp.info - acheteur", name="Acheteur", order=5, diff --git a/src/pages/fournisseur.py b/src/pages/fournisseur.py index 509ca1f..077d276 100644 --- a/src/pages/fournisseur.py +++ b/src/pages/fournisseur.py @@ -1,13 +1,273 @@ -from dash import html, register_page +import datetime + +import polars as pl +from dash import Input, Output, State, callback, dash_table, dcc, html, register_page + +from src.figures import point_on_map +from src.utils import format_number, get_annuaire_data, get_departement_region, lf register_page( __name__, - path_template="/fournisseur/", + path_template="/fournisseurs/", title="decp.info - fournisseur", name="Fournisseur", order=5, ) +# 21690123100011 + layout = [ - html.Div(className="container", children=["Cette page est encore en construction."]) + dcc.Store(id="fournisseur_data", storage_type="memory"), + dcc.Location(id="url", refresh="callback-nav"), + html.Div( + className="container", + children=[ + html.Div( + className="wrapper", + children=[ + html.H2( + className="org_title", + children=[ + html.Span(id="fournisseur_siret"), + " - ", + html.Span(id="fournisseur_nom"), + ], + ), + html.Div( + className="org_year", + children=dcc.Dropdown( + id="fournisseur_year", + options=["Toutes"] + + [ + str(year) + for year in range( + 2018, int(datetime.date.today().year) + 1 + ) + ], + placeholder="Année", + ), + ), + html.Div( + className="org_infos", + children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. + html.P( + ["Commune : ", html.Strong(id="fournisseur_commune")] + ), + html.P( + [ + "Département : ", + html.Strong(id="fournisseur_departement"), + ] + ), + html.P(["Région : ", html.Strong(id="fournisseur_region")]), + html.A( + id="fournisseur_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + target="_blank", + ), + ], + ), + html.Div( + className="org_stats", + children=[ + html.P(id="fournisseur_titre_stats"), + html.P(id="fournisseur_marches_remportes"), + html.P(id="fournisseur_acheteurs_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-fournisseur-data", + ), + dcc.Download(id="download-fournisseur-data"), + ], + ), + html.Div(className="org_map", id="fournisseur_map"), + ], + ), + # récupérer les données de l'acheteur sur l'api annuaire + html.H3("Derniers marchés publics remportés"), + html.Div(id="fournisseur_last_marches", children=""), + ], + ), ] + + +@callback( + Output(component_id="fournisseur_siret", component_property="children"), + Output(component_id="fournisseur_nom", component_property="children"), + Output(component_id="fournisseur_commune", component_property="children"), + Output(component_id="fournisseur_map", component_property="children"), + Output(component_id="fournisseur_departement", component_property="children"), + Output(component_id="fournisseur_region", component_property="children"), + Output(component_id="fournisseur_lien_annuaire", component_property="href"), + Input(component_id="url", component_property="pathname"), +) +def update_fournisseur_infos(url): + fournisseur_siret = url.split("/")[-1] + if len(fournisseur_siret) != 14: + fournisseur_siret = ( + f"Le SIRET renseigné doit faire 14 caractères ({fournisseur_siret})" + ) + data = get_annuaire_data(fournisseur_siret) + data_etablissement = data["matching_etablissements"][0] + fournisseur_map = point_on_map( + data_etablissement["latitude"], data_etablissement["longitude"] + ) + code_departement, nom_departement, nom_region = get_departement_region( + data_etablissement["code_postal"] + ) + departement = f"{nom_departement} ({code_departement})" + lien_annuaire = ( + f"https://annuaire-entreprises.data.gouv.fr/etablissement/{fournisseur_siret}" + ) + return ( + fournisseur_siret, + data["nom_raison_sociale"], + data_etablissement["libelle_commune"], + fournisseur_map, + departement, + nom_region, + lien_annuaire, + ) + + +@callback( + Output(component_id="fournisseur_marches_remportes", component_property="children"), + Output( + component_id="fournisseur_acheteurs_differents", component_property="children" + ), + Input(component_id="fournisseur_data", component_property="data"), +) +def update_fournisseur_stats(data): + df = pl.DataFrame(data) + if df.height == 0: + df = pl.DataFrame(schema=lf.collect_schema()) + df_marches = df.unique("uid") + nb_marches = format_number(df_marches.height) + # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) + marches_remportes = [html.Strong(nb_marches), " marchés et accord-cadres remportés"] + # + ", pour un total de ", html.Strong(somme_marches + " €")] + del df_marches + + nb_acheteurs = df.unique("acheteur_id").height + nb_acheteurs = [ + html.Strong(format_number(nb_acheteurs)), + " fournisseurs (SIRET) différents", + ] + del df + + return marches_remportes, nb_acheteurs + + +@callback( + Output(component_id="fournisseur_data", component_property="data"), + Input(component_id="url", component_property="pathname"), + Input(component_id="fournisseur_year", component_property="value"), +) +def get_fournisseur_marches_data(url, fournisseur_year: str) -> pl.LazyFrame: + fournisseur_siret = url.split("/")[-1] + lff = lf.filter( + (pl.col("titulaire_id") == fournisseur_siret) + & (pl.col("titulaire_typeIdentifiant") == "SIRET") + ) + lff = lff.fill_null("") + lff = lff.select( + "uid", + "objet", + "dateNotification", + "acheteur_id", + "acheteur_nom", + "montant", + "codeCPV", + "dureeMois", + ) + if fournisseur_year and fournisseur_year != "Toutes": + lff = lff.filter( + pl.col("dateNotification").cast(pl.String).str.starts_with(fournisseur_year) + ) + lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) + + data = lff.collect(engine="streaming").to_dicts() + return data + + +@callback( + Output(component_id="fournisseur_last_marches", component_property="children"), + Input(component_id="fournisseur_data", component_property="data"), +) +def get_last_marches_table(data) -> html.Div: + columns = [ + "uid", + "objet", + "dateNotification", + "acheteur_nom", + "montant", + "codeCPV", + "dureeMois", + ] + + table = html.Div( + className="marches_table", + id="marches_datatable", + children=dash_table.DataTable( + data=data, + page_action="native", + columns=[ + { + "name": i, + "id": i, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": False, + } + for i in columns + if i not in ["titulaire_id"] + ], + page_size=10, + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "300px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "titulaire_nom"}, + "minWidth": "200px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + ), + ) + return table + + +@callback( + Output("download-fournisseur-data", "data"), + Input("btn-download-fournisseur-data", "n_clicks"), + State(component_id="fournisseur_data", component_property="data"), + State(component_id="fournisseur_nom", component_property="children"), + State(component_id="fournisseur_year", component_property="value"), + prevent_initial_call=True, +) +def download_fournisseur_data( + n_clicks, + data: [dict], + fournisseur_nom: str, + annee: str, +): + df_to_download = pl.DataFrame(data) + + def to_bytes(buffer): + df_to_download.write_excel( + buffer, worksheet="DECP" if annee in ["Toutes", None] else annee + ) + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes(to_bytes, filename=f"decp_{fournisseur_nom}_{date}.xlsx") diff --git a/src/utils.py b/src/utils.py index 96c2bfb..452a618 100644 --- a/src/utils.py +++ b/src/utils.py @@ -56,7 +56,7 @@ def add_org_links(dff: pl.DataFrame): dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( - '' + pl.col("titulaire_id") @@ -67,7 +67,7 @@ def add_org_links(dff: pl.DataFrame): ) dff = dff.with_columns( ( - '' + pl.col("acheteur_id") From 4081b369d9f854da3451d9d3b3afbba539ef2374 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 19:16:50 +0200 Subject: [PATCH 19/41] Recherche possible dans les dates (comme des strings) #37 --- src/utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 452a618..2a7b165 100644 --- a/src/utils.py +++ b/src/utils.py @@ -98,6 +98,14 @@ def numbers_to_strings(lff: pl.LazyFrame) -> pl.LazyFrame: return lff +def dates_to_strings(lff: pl.LazyFrame, column: str) -> pl.LazyFrame: + """ + Convert a date column to string type. + """ + lff = lff.with_columns(pl.col(column).cast(pl.String).fill_null("")) + return lff + + def format_number(number) -> str: number = "{:,}".format(number).replace(",", " ") return number @@ -167,6 +175,10 @@ def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: print("filter_value_type:", type(filter_value)) print("col_type:", col_type) + if col_type == "Date": + # Convertir la colonne en chaînes de caractères + lff = dates_to_strings(lff, col_name) + if operator in ("<", "<=", ">", ">="): filter_value = int(filter_value) if operator == "<": @@ -186,7 +198,7 @@ def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: continue lff = lff.filter(pl.col(col_name) == filter_value) - elif operator == "contains" and col_type == "String": + elif operator == "contains" and col_type in ["String", "Date"]: lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) # elif operator == 'datestartswith': From c1955ac807a94d6d194778a6a7544c09f0f2875a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 20:13:20 +0200 Subject: [PATCH 20/41] =?UTF-8?q?Ajout=20des=20balises=20meta=20og=20et=20?= =?UTF-8?q?twitter=20(=C3=A0=20tester)=20#39?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app.py | 40 ++++++++++++++++++++++++++++++++++++++++ src/pages/about.py | 3 --- src/pages/home.py | 11 ++++++++--- src/utils.py | 3 --- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/app.py b/src/app.py index 66e5f30..431c2d7 100644 --- a/src/app.py +++ b/src/app.py @@ -1,14 +1,54 @@ import logging +import os import dash_bootstrap_components as dbc from dash import Dash, dcc, html, page_container, page_registry +from dotenv import load_dotenv from flask import send_from_directory +load_dotenv() + +domain_name = ( + "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" +) +image_url = f"https://{domain_name}/assets/decp.info.png" +title = "decp.info - exploration des marchés publics français" +description = ( + "Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. " + "Pour une commande publique accessible à toutes et tous." +) + app = Dash( external_stylesheets=[dbc.themes.SIMPLEX], title="decp.info", use_pages=True, compress=True, + meta_tags=[ + { + "property": "og:image", + "content": image_url, + }, + { + "name": "twitter:image", + "content": image_url, + }, + { + "property": "og:title", + "content": title, + }, + { + "name": "twitter:title", + "content": title, + }, + { + "property": "og:description", + "content": description, + }, + { + "name": "twitter:description", + "content": description, + }, + ], ) # COSMO (belle font, blue), # UNITED (rouge, ubuntu font), diff --git a/src/pages/about.py b/src/pages/about.py index bdae8df..bf2ce2d 100644 --- a/src/pages/about.py +++ b/src/pages/about.py @@ -1,14 +1,11 @@ import os from dash import dcc, html, register_page -from dotenv import load_dotenv from src.figures import get_sources_tables title = "À propos" -load_dotenv() - register_page( __name__, path="/a-propos", title=f"decp.info - {title}", name=title, order=5 ) diff --git a/src/pages/home.py b/src/pages/home.py index 9e19d8c..d4df7c1 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -3,7 +3,6 @@ from datetime import datetime import polars as pl from dash import Input, Output, State, callback, dash_table, dcc, html, register_page -from dotenv import load_dotenv from src.utils import ( add_org_links, @@ -14,8 +13,6 @@ from src.utils import ( sort_table_data, ) -load_dotenv() - update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") @@ -56,6 +53,14 @@ datatable = html.Div( "lineHeight": "14px", "whiteSpace": "normal", }, + { + "if": {"column_id": "titulaire_nom"}, + "minWidth": "250px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, ], data_timestamp=0, markdown_options={"html": True}, diff --git a/src/utils.py b/src/utils.py index 2a7b165..31fdb73 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,12 +5,9 @@ from time import sleep import polars as pl import polars.selectors as cs -from dotenv import load_dotenv from httpx import get from polars.exceptions import ComputeError -load_dotenv() - operators = [ ["s<", "<"], ["s>", ">"], From 0db814710b88d188a3546e1b796c34d0f0bd5262 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 20:21:26 +0200 Subject: [PATCH 21/41] Changelog --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 70a2fae..d5efcc1 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ python run.py #### 2.1.0 -- Ajout de la vue acheteur +- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) et [fournisseur](https://decp.info/fournisseurs/51903758414786) ([#28](https://github.com/ColinMaudry/decp.info/issues/28) et [#35](https://github.com/ColinMaudry/decp.info/issues/35)) +- Ajout des balises HTML meta Open Graph et Twitter ([#39](https://github.com/ColinMaudry/decp.info/issues/39)) pour de beaux aperçus de liens - Variables globales uniquement en lecture ##### 2.0.1 (23 septembre 2025) From d00efcd500ae86d2f9b6b2f3306ce522de939ef0 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 20:40:35 +0200 Subject: [PATCH 22/41] =?UTF-8?q?Script=20de=20d=C3=A9ploiement=20int?= =?UTF-8?q?=C3=A9gr=C3=A9=20au=20yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yaml | 10 +++++++++- deploy.sh | 15 --------------- 2 files changed, 9 insertions(+), 16 deletions(-) delete mode 100755 deploy.sh diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index c2365fb..9d9f9e4 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -40,4 +40,12 @@ jobs: key: ${{ secrets.ARTIFACT_SSH_KEY }} passphrase: ${{ secrets.SSH_PSWD }} command_timeout: 5m - script: ${{ secrets.APP_PATH }}/deploy.sh ${{ env.APP_NAME }} + script: | + systemctl stop ${{ vars.APP_NAME }} + cd /var/www/${{ vars.APP_NAME }} + git pull + source .venv/bin/activate + pip install . + deactivate + chown -R ${{ vars.APP_NAME }}:www-data * + systemctl start ${{ vars.APP_NAME }} diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 16f3723..0000000 --- a/deploy.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!bin/bash - -# Utilisé principalement avec les Github Actions -# cf. .github/workflows/deploy.yaml - -appname = "$1" - -systemctl stop $appname -cd /var/www/$appname -git pull -source .venv/bin/activate -pip install . -deactivate -chown -R $appname:www-data * -systemctl start $appname From d9db156626e0836743d37a62cfed5066112f0f40 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 21:01:22 +0200 Subject: [PATCH 23/41] Liens entre pages acheteur et titulaire, fournisseur => titulaire #28 #35 --- README.md | 2 +- src/pages/about.py | 2 +- src/pages/acheteur.py | 27 +++-- src/pages/{fournisseur.py => titulaire.py} | 128 +++++++++++---------- src/utils.py | 11 ++ 5 files changed, 99 insertions(+), 71 deletions(-) rename src/pages/{fournisseur.py => titulaire.py} (63%) diff --git a/README.md b/README.md index d5efcc1..e87977d 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ python run.py #### 2.1.0 -- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) et [fournisseur](https://decp.info/fournisseurs/51903758414786) ([#28](https://github.com/ColinMaudry/decp.info/issues/28) et [#35](https://github.com/ColinMaudry/decp.info/issues/35)) +- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) et [titulaire](https://decp.info/titulaires/51903758414786) ([#28](https://github.com/ColinMaudry/decp.info/issues/28) et [#35](https://github.com/ColinMaudry/decp.info/issues/35)) - Ajout des balises HTML meta Open Graph et Twitter ([#39](https://github.com/ColinMaudry/decp.info/issues/39)) pour de beaux aperçus de liens - Variables globales uniquement en lecture diff --git a/src/pages/about.py b/src/pages/about.py index bf2ce2d..f7eae51 100644 --- a/src/pages/about.py +++ b/src/pages/about.py @@ -24,7 +24,7 @@ j'aimerais beaucoup échanger avec vous pour comprendre vos cas d'usages et vos En effet, le potentiel des données d'attribution de marchés et des données qui peuvent les enrichir est très loin d'être exploité par les fonctionnalités actuelles de decp.info. Il est ainsi possible de rajouter -- de nombreuses visualisations de données (cartes, graphiques, tableaux) sur des thématiques variées (vivacité de la concurrence, secteurs d'activité, insertion par l'activité économique (IAE), distance acheteur-fournisseur...) +- de nombreuses visualisations de données (cartes, graphiques, tableaux) sur des thématiques variées (vivacité de la concurrence, secteurs d'activité, insertion par l'activité économique (IAE), distance acheteur-titulaire...) - la sauvegarde de filtres pour les retrouver plus tard et les partager - des alertes par email si des marchés correspondant à certains critères - le développement d'une API pour alimenter d'autres logiciels diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index a70ff04..f47c87c 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -4,7 +4,13 @@ import polars as pl from dash import Input, Output, State, callback, dash_table, dcc, html, register_page from src.figures import point_on_map -from src.utils import format_number, get_annuaire_data, get_departement_region, lf +from src.utils import ( + add_org_links_in_dict, + format_number, + get_annuaire_data, + get_departement_region, + lf, +) register_page( __name__, @@ -71,7 +77,7 @@ layout = [ children=[ html.P(id="acheteur_titre_stats"), html.P(id="acheteur_marches_attribues"), - html.P(id="acheteur_fournisseurs_differents"), + html.P(id="acheteur_titulaires_differents"), html.Button( "Téléchargement au format Excel", id="btn-download-acheteur-data", @@ -132,7 +138,7 @@ def update_acheteur_infos(url): @callback( Output(component_id="acheteur_marches_attribues", component_property="children"), Output( - component_id="acheteur_fournisseurs_differents", component_property="children" + component_id="acheteur_titulaires_differents", component_property="children" ), Input(component_id="acheteur_data", component_property="data"), ) @@ -147,14 +153,14 @@ def update_acheteur_stats(data): # + ", pour un total de ", html.Strong(somme_marches + " €")] del df_marches - nb_fournisseurs = df.unique("titulaire_id").height - nb_fournisseurs = [ - html.Strong(format_number(nb_fournisseurs)), - " fournisseurs (SIRET) différents", + nb_titulaires = df.unique("titulaire_id").height + nb_titulaires = [ + html.Strong(format_number(nb_titulaires)), + " titulaires (SIRET) différents", ] del df - return marches_attribues, nb_fournisseurs + return marches_attribues, nb_titulaires @callback( @@ -201,11 +207,14 @@ def get_last_marches_table(data) -> html.Div: "dureeMois", ] + data = add_org_links_in_dict(data, "titulaire") + table = html.Div( className="marches_table", id="marches_datatable", children=dash_table.DataTable( data=data, + markdown_options={"html": True}, page_action="native", columns=[ { @@ -234,7 +243,7 @@ def get_last_marches_table(data) -> html.Div: "minWidth": "200px", "textAlign": "left", "overflow": "hidden", - "lineHeight": "14px", + "lineHeight": "18px", "whiteSpace": "normal", }, ], diff --git a/src/pages/fournisseur.py b/src/pages/titulaire.py similarity index 63% rename from src/pages/fournisseur.py rename to src/pages/titulaire.py index 077d276..0a90dc2 100644 --- a/src/pages/fournisseur.py +++ b/src/pages/titulaire.py @@ -4,12 +4,18 @@ import polars as pl from dash import Input, Output, State, callback, dash_table, dcc, html, register_page from src.figures import point_on_map -from src.utils import format_number, get_annuaire_data, get_departement_region, lf +from src.utils import ( + add_org_links_in_dict, + format_number, + get_annuaire_data, + get_departement_region, + lf, +) register_page( __name__, - path_template="/fournisseurs/", - title="decp.info - fournisseur", + path_template="/titulaires/", + title="decp.info - titulaire", name="Fournisseur", order=5, ) @@ -17,7 +23,7 @@ register_page( # 21690123100011 layout = [ - dcc.Store(id="fournisseur_data", storage_type="memory"), + dcc.Store(id="titulaire_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), html.Div( className="container", @@ -28,15 +34,15 @@ layout = [ html.H2( className="org_title", children=[ - html.Span(id="fournisseur_siret"), + html.Span(id="titulaire_siret"), " - ", - html.Span(id="fournisseur_nom"), + html.Span(id="titulaire_nom"), ], ), html.Div( className="org_year", children=dcc.Dropdown( - id="fournisseur_year", + id="titulaire_year", options=["Toutes"] + [ str(year) @@ -51,18 +57,16 @@ layout = [ className="org_infos", children=[ # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. - html.P( - ["Commune : ", html.Strong(id="fournisseur_commune")] - ), + html.P(["Commune : ", html.Strong(id="titulaire_commune")]), html.P( [ "Département : ", - html.Strong(id="fournisseur_departement"), + html.Strong(id="titulaire_departement"), ] ), - html.P(["Région : ", html.Strong(id="fournisseur_region")]), + html.P(["Région : ", html.Strong(id="titulaire_region")]), html.A( - id="fournisseur_lien_annuaire", + id="titulaire_lien_annuaire", children="Plus de détails sur l'Annuaire des entreprises", target="_blank", ), @@ -71,46 +75,46 @@ layout = [ html.Div( className="org_stats", children=[ - html.P(id="fournisseur_titre_stats"), - html.P(id="fournisseur_marches_remportes"), - html.P(id="fournisseur_acheteurs_differents"), + html.P(id="titulaire_titre_stats"), + html.P(id="titulaire_marches_remportes"), + html.P(id="titulaire_acheteurs_differents"), html.Button( "Téléchargement au format Excel", - id="btn-download-fournisseur-data", + id="btn-download-titulaire-data", ), - dcc.Download(id="download-fournisseur-data"), + dcc.Download(id="download-titulaire-data"), ], ), - html.Div(className="org_map", id="fournisseur_map"), + html.Div(className="org_map", id="titulaire_map"), ], ), # récupérer les données de l'acheteur sur l'api annuaire html.H3("Derniers marchés publics remportés"), - html.Div(id="fournisseur_last_marches", children=""), + html.Div(id="titulaire_last_marches", children=""), ], ), ] @callback( - Output(component_id="fournisseur_siret", component_property="children"), - Output(component_id="fournisseur_nom", component_property="children"), - Output(component_id="fournisseur_commune", component_property="children"), - Output(component_id="fournisseur_map", component_property="children"), - Output(component_id="fournisseur_departement", component_property="children"), - Output(component_id="fournisseur_region", component_property="children"), - Output(component_id="fournisseur_lien_annuaire", component_property="href"), + Output(component_id="titulaire_siret", component_property="children"), + Output(component_id="titulaire_nom", component_property="children"), + Output(component_id="titulaire_commune", component_property="children"), + Output(component_id="titulaire_map", component_property="children"), + Output(component_id="titulaire_departement", component_property="children"), + Output(component_id="titulaire_region", component_property="children"), + Output(component_id="titulaire_lien_annuaire", component_property="href"), Input(component_id="url", component_property="pathname"), ) -def update_fournisseur_infos(url): - fournisseur_siret = url.split("/")[-1] - if len(fournisseur_siret) != 14: - fournisseur_siret = ( - f"Le SIRET renseigné doit faire 14 caractères ({fournisseur_siret})" +def update_titulaire_infos(url): + titulaire_siret = url.split("/")[-1] + if len(titulaire_siret) != 14: + titulaire_siret = ( + f"Le SIRET renseigné doit faire 14 caractères ({titulaire_siret})" ) - data = get_annuaire_data(fournisseur_siret) + data = get_annuaire_data(titulaire_siret) data_etablissement = data["matching_etablissements"][0] - fournisseur_map = point_on_map( + titulaire_map = point_on_map( data_etablissement["latitude"], data_etablissement["longitude"] ) code_departement, nom_departement, nom_region = get_departement_region( @@ -118,13 +122,13 @@ def update_fournisseur_infos(url): ) departement = f"{nom_departement} ({code_departement})" lien_annuaire = ( - f"https://annuaire-entreprises.data.gouv.fr/etablissement/{fournisseur_siret}" + f"https://annuaire-entreprises.data.gouv.fr/etablissement/{titulaire_siret}" ) return ( - fournisseur_siret, + titulaire_siret, data["nom_raison_sociale"], data_etablissement["libelle_commune"], - fournisseur_map, + titulaire_map, departement, nom_region, lien_annuaire, @@ -132,13 +136,13 @@ def update_fournisseur_infos(url): @callback( - Output(component_id="fournisseur_marches_remportes", component_property="children"), + Output(component_id="titulaire_marches_remportes", component_property="children"), Output( - component_id="fournisseur_acheteurs_differents", component_property="children" + component_id="titulaire_acheteurs_differents", component_property="children" ), - Input(component_id="fournisseur_data", component_property="data"), + Input(component_id="titulaire_data", component_property="data"), ) -def update_fournisseur_stats(data): +def update_titulaire_stats(data): df = pl.DataFrame(data) if df.height == 0: df = pl.DataFrame(schema=lf.collect_schema()) @@ -152,7 +156,7 @@ def update_fournisseur_stats(data): nb_acheteurs = df.unique("acheteur_id").height nb_acheteurs = [ html.Strong(format_number(nb_acheteurs)), - " fournisseurs (SIRET) différents", + " titulaires (SIRET) différents", ] del df @@ -160,14 +164,14 @@ def update_fournisseur_stats(data): @callback( - Output(component_id="fournisseur_data", component_property="data"), + Output(component_id="titulaire_data", component_property="data"), Input(component_id="url", component_property="pathname"), - Input(component_id="fournisseur_year", component_property="value"), + Input(component_id="titulaire_year", component_property="value"), ) -def get_fournisseur_marches_data(url, fournisseur_year: str) -> pl.LazyFrame: - fournisseur_siret = url.split("/")[-1] +def get_titulaire_marches_data(url, titulaire_year: str) -> pl.LazyFrame: + titulaire_siret = url.split("/")[-1] lff = lf.filter( - (pl.col("titulaire_id") == fournisseur_siret) + (pl.col("titulaire_id") == titulaire_siret) & (pl.col("titulaire_typeIdentifiant") == "SIRET") ) lff = lff.fill_null("") @@ -181,9 +185,9 @@ def get_fournisseur_marches_data(url, fournisseur_year: str) -> pl.LazyFrame: "codeCPV", "dureeMois", ) - if fournisseur_year and fournisseur_year != "Toutes": + if titulaire_year and titulaire_year != "Toutes": lff = lff.filter( - pl.col("dateNotification").cast(pl.String).str.starts_with(fournisseur_year) + pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year) ) lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True) @@ -192,8 +196,8 @@ def get_fournisseur_marches_data(url, fournisseur_year: str) -> pl.LazyFrame: @callback( - Output(component_id="fournisseur_last_marches", component_property="children"), - Input(component_id="fournisseur_data", component_property="data"), + Output(component_id="titulaire_last_marches", component_property="children"), + Input(component_id="titulaire_data", component_property="data"), ) def get_last_marches_table(data) -> html.Div: columns = [ @@ -206,11 +210,15 @@ def get_last_marches_table(data) -> html.Div: "dureeMois", ] + # Ajout des liens vers les acheteurs + data = add_org_links_in_dict(data, "acheteur") + table = html.Div( className="marches_table", id="marches_datatable", children=dash_table.DataTable( data=data, + markdown_options={"html": True}, page_action="native", columns=[ { @@ -239,7 +247,7 @@ def get_last_marches_table(data) -> html.Div: "minWidth": "200px", "textAlign": "left", "overflow": "hidden", - "lineHeight": "14px", + "lineHeight": "18px", "whiteSpace": "normal", }, ], @@ -249,17 +257,17 @@ def get_last_marches_table(data) -> html.Div: @callback( - Output("download-fournisseur-data", "data"), - Input("btn-download-fournisseur-data", "n_clicks"), - State(component_id="fournisseur_data", component_property="data"), - State(component_id="fournisseur_nom", component_property="children"), - State(component_id="fournisseur_year", component_property="value"), + Output("download-titulaire-data", "data"), + Input("btn-download-titulaire-data", "n_clicks"), + State(component_id="titulaire_data", component_property="data"), + State(component_id="titulaire_nom", component_property="children"), + State(component_id="titulaire_year", component_property="value"), prevent_initial_call=True, ) -def download_fournisseur_data( +def download_titulaire_data( n_clicks, data: [dict], - fournisseur_nom: str, + titulaire_nom: str, annee: str, ): df_to_download = pl.DataFrame(data) @@ -270,4 +278,4 @@ def download_fournisseur_data( ) date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") - return dcc.send_bytes(to_bytes, filename=f"decp_{fournisseur_nom}_{date}.xlsx") + return dcc.send_bytes(to_bytes, filename=f"decp_{titulaire_nom}_{date}.xlsx") diff --git a/src/utils.py b/src/utils.py index 31fdb73..980c6fe 100644 --- a/src/utils.py +++ b/src/utils.py @@ -74,6 +74,17 @@ def add_org_links(dff: pl.DataFrame): return dff +def add_org_links_in_dict(data: list, org_type: str) -> list: + new_data = [] + for marche in data: + org_id = marche[org_type + "_id"] + marche[org_type + "_nom"] = ( + f'{marche[org_type + "_nom"]}' + ) + new_data.append(marche) + return new_data + + def booleans_to_strings(lff: pl.LazyFrame) -> pl.LazyFrame: """ Convert all boolean columns to string type. From 4895bfc5ac17c332286a8e7a48f5f531fdbe6a88 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 21:05:23 +0200 Subject: [PATCH 24/41] Ajout des filtres natifs #28 #35 --- src/pages/acheteur.py | 1 + src/pages/titulaire.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index f47c87c..89f234c 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -216,6 +216,7 @@ def get_last_marches_table(data) -> html.Div: data=data, markdown_options={"html": True}, page_action="native", + filter_action="native", columns=[ { "name": i, diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 0a90dc2..231eb45 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -220,6 +220,7 @@ def get_last_marches_table(data) -> html.Div: data=data, markdown_options={"html": True}, page_action="native", + filter_action="native", columns=[ { "name": i, From 70c5b91e6da75880b2d55bc2eb6740ecd94eeafb Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 21:27:40 +0200 Subject: [PATCH 25/41] Configuration des filtres (case insensitive, placeholder text #28 #35 --- src/pages/acheteur.py | 1 + src/pages/titulaire.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 89f234c..4d54052 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -217,6 +217,7 @@ def get_last_marches_table(data) -> html.Div: markdown_options={"html": True}, page_action="native", filter_action="native", + filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, columns=[ { "name": i, diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 231eb45..590d823 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -221,6 +221,7 @@ def get_last_marches_table(data) -> html.Div: markdown_options={"html": True}, page_action="native", filter_action="native", + filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, columns=[ { "name": i, @@ -244,7 +245,7 @@ def get_last_marches_table(data) -> html.Div: "whiteSpace": "normal", }, { - "if": {"column_id": "titulaire_nom"}, + "if": {"column_id": "acheteur_nom"}, "minWidth": "200px", "textAlign": "left", "overflow": "hidden", From ce9d23100d5c4d39c0e3f0319ca9c66f689b06ec Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 22:08:03 +0200 Subject: [PATCH 26/41] Correction des balises meta #39 --- src/app.py | 39 +-------------------------------------- src/pages/about.py | 13 ++++++++++--- src/pages/acheteur.py | 5 ++++- src/pages/home.py | 14 +++++++++++--- src/pages/statistiques.py | 14 ++++++++++---- src/pages/titulaire.py | 9 ++++++--- src/utils.py | 11 +++++++++++ 7 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/app.py b/src/app.py index 431c2d7..da958c6 100644 --- a/src/app.py +++ b/src/app.py @@ -1,5 +1,4 @@ import logging -import os import dash_bootstrap_components as dbc from dash import Dash, dcc, html, page_container, page_registry @@ -8,47 +7,11 @@ from flask import send_from_directory load_dotenv() -domain_name = ( - "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" -) -image_url = f"https://{domain_name}/assets/decp.info.png" -title = "decp.info - exploration des marchés publics français" -description = ( - "Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. " - "Pour une commande publique accessible à toutes et tous." -) - app = Dash( external_stylesheets=[dbc.themes.SIMPLEX], title="decp.info", use_pages=True, compress=True, - meta_tags=[ - { - "property": "og:image", - "content": image_url, - }, - { - "name": "twitter:image", - "content": image_url, - }, - { - "property": "og:title", - "content": title, - }, - { - "name": "twitter:title", - "content": title, - }, - { - "property": "og:description", - "content": description, - }, - { - "name": "twitter:description", - "content": description, - }, - ], ) # COSMO (belle font, blue), # UNITED (rouge, ubuntu font), @@ -71,7 +34,7 @@ logging.basicConfig( app.index_string = """ - + {%metas%} {%title%} diff --git a/src/pages/about.py b/src/pages/about.py index f7eae51..b44a7ee 100644 --- a/src/pages/about.py +++ b/src/pages/about.py @@ -3,18 +3,25 @@ import os from dash import dcc, html, register_page from src.figures import get_sources_tables +from src.utils import meta_content -title = "À propos" +name = "À propos" register_page( - __name__, path="/a-propos", title=f"decp.info - {title}", name=title, order=5 + __name__, + path="/a-propos", + title=meta_content["title"], + name=name, + description=meta_content["description"], + image_url=meta_content["image_url"], + order=5, ) layout = [ html.Div( className="container", children=[ - html.H2(title), + html.H2(name), dcc.Markdown( """Outil d'exploration libre et gratuit des données de marchés publics, développé par Colin Maudry. diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 4d54052..1b3f202 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -10,13 +10,16 @@ from src.utils import ( get_annuaire_data, get_departement_region, lf, + meta_content, ) register_page( __name__, path_template="/acheteurs/", - title="decp.info - acheteur", + title=meta_content["title"], name="Acheteur", + description=meta_content["description"], + image_url=meta_content["image_url"], order=5, ) diff --git a/src/pages/home.py b/src/pages/home.py index d4df7c1..48dd24e 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -10,6 +10,7 @@ from src.utils import ( filter_table_data, format_number, lf, + meta_content, sort_table_data, ) @@ -18,9 +19,16 @@ update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") schema = lf.collect_schema() - -title = "Tableau" -register_page(__name__, path="/", title="decp.info", name=title, order=1) +name = "Tableau" +register_page( + __name__, + path="/", + title=meta_content["title"], + name=name, + description=meta_content["description"], + image_url=meta_content["image_url"], + order=1, +) datatable = html.Div( className="marches_table", diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 877457d..a55b8c8 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -1,12 +1,18 @@ from dash import dcc, html, register_page from src.figures import get_barchart_sources, get_map_count_marches -from src.utils import lf +from src.utils import lf, meta_content -title = "Statistiques" +name = "Statistiques" register_page( - __name__, path="/statistiques", title=f"decp.info - {title}", name=title, order=3 + __name__, + path="/statistiques", + title=meta_content["title"], + name=name, + description=meta_content["description"], + image_url=meta_content["image_url"], + order=3, ) @@ -14,7 +20,7 @@ layout = [ html.Div( className="container", children=[ - html.H2(title), + html.H2(name), dcc.Loading( overlay_style={"visibility": "visible", "filter": "blur(2px)"}, id="loading-statistques", diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 590d823..b0d1aa6 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -10,13 +10,16 @@ from src.utils import ( get_annuaire_data, get_departement_region, lf, + meta_content, ) register_page( __name__, path_template="/titulaires/", - title="decp.info - titulaire", - name="Fournisseur", + title=meta_content["title"], + name="Titulaire", + description=meta_content["description"], + image_url=meta_content["image_url"], order=5, ) @@ -245,7 +248,7 @@ def get_last_marches_table(data) -> html.Div: "whiteSpace": "normal", }, { - "if": {"column_id": "acheteur_nom"}, + "if": {"column_id": " acheteur_nom"}, "minWidth": "200px", "textAlign": "left", "overflow": "hidden", diff --git a/src/utils.py b/src/utils.py index 980c6fe..454780c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -227,3 +227,14 @@ def sort_table_data(lff: pl.LazyFrame, sort_by: list) -> pl.LazyFrame: lf = get_decp_data() departements = get_departements() +domain_name = ( + "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" +) +meta_content = { + "image_url": f"https://{domain_name}/assets/decp.info.png", + "title": "decp.info - exploration des marchés publics français", + "description": ( + "Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. " + "Pour une commande publique accessible à toutes et tous." + ), +} From 4394f38e35b8653c5d82a0ed2948909b61ad6ba6 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 22:19:07 +0200 Subject: [PATCH 27/41] Fix link titulaire --- src/assets/style.css | 2 +- src/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index 873998a..a04257a 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -118,7 +118,7 @@ summary > h3 { padding-top: 28px; } -/* Vue acheteur/fournisseur */ +/* Vue acheteur/titulaire */ .wrapper { display: grid; grid-gap: 10px; diff --git a/src/utils.py b/src/utils.py index 454780c..d56b031 100644 --- a/src/utils.py +++ b/src/utils.py @@ -53,7 +53,7 @@ def add_org_links(dff: pl.DataFrame): dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( - '' + pl.col("titulaire_id") From fdb38b3b9bc69a98826512d4767b3ca36f7dd981 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 28 Sep 2025 23:11:09 +0200 Subject: [PATCH 28/41] Formatage des montants #45 --- src/pages/acheteur.py | 7 ++++++- src/pages/home.py | 4 ++++ src/pages/titulaire.py | 7 ++++++- src/utils.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 1b3f202..54c3ada 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -6,6 +6,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.figures import point_on_map from src.utils import ( add_org_links_in_dict, + format_montant, format_number, get_annuaire_data, get_departement_region, @@ -180,6 +181,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: "objet", "dateNotification", "titulaire_id", + "titulaire_typeIdentifiant", "titulaire_nom", "montant", "codeCPV", @@ -210,6 +212,9 @@ def get_last_marches_table(data) -> html.Div: "dureeMois", ] + dff = pl.DataFrame(data) + dff = format_montant(dff) + data = dff.to_dicts() data = add_org_links_in_dict(data, "titulaire") table = html.Div( @@ -231,7 +236,7 @@ def get_last_marches_table(data) -> html.Div: "hideable": False, } for i in columns - if i not in ["titulaire_id"] + if i not in ["titulaire_id", "titulaire_typeIdentifiant"] ], page_size=10, style_cell_conditional=[ diff --git a/src/pages/home.py b/src/pages/home.py index 48dd24e..ede5c74 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -8,6 +8,7 @@ from src.utils import ( add_org_links, add_resource_link, filter_table_data, + format_montant, format_number, lf, meta_content, @@ -184,6 +185,9 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) # Ajout des liens vers les fichiers Open Data dff = add_resource_link(dff) + # Formatage des montants + dff = format_montant(dff) + # Liste finale de colonnes columns = [ { diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index b0d1aa6..7ea1a2a 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -6,6 +6,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.figures import point_on_map from src.utils import ( add_org_links_in_dict, + format_montant, format_number, get_annuaire_data, get_departement_region, @@ -213,7 +214,11 @@ def get_last_marches_table(data) -> html.Div: "dureeMois", ] - # Ajout des liens vers les acheteurs + dff = pl.DataFrame(data) + dff = format_montant(dff) + data = dff.to_dicts() + # Idéalement on utiliserait add_org_links(), mais le résultat attendu + # est différent de home.py (Tableau) data = add_org_links_in_dict(data, "acheteur") table = html.Div( diff --git a/src/utils.py b/src/utils.py index d56b031..d49a7f5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -119,6 +119,34 @@ def format_number(number) -> str: return number +def format_montant(dff: pl.DataFrame) -> pl.DataFrame: + def format_function(expr, scale=None): + # https://stackoverflow.com/a/78636786 + expr = expr.cast(pl.String).str.splitn(".", 2) + + num = expr.struct[0] + frac = expr.struct[1] + + # Ajout des espaces + num = ( + num.str.reverse() + .str.replace_all(r"\d{3}", "$0 ") + .str.reverse() + .str.replace(r"^ ", "") + ) + + frac: pl.Expr = ( + pl.when(frac.is_not_null() & ~frac.is_in(["0"])) + .then("," + frac) + .otherwise(pl.lit("")) + ) + + return num + frac + pl.lit(" €") + + dff = dff.with_columns(pl.col("montant").pipe(format_function).alias("montant")) + return dff + + def get_annuaire_data(siret: str) -> dict: url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" response = get(url) From 502f652791262c5962ca7022d1ad5cc2d25fafcb Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 17:03:44 +0200 Subject: [PATCH 29/41] =?UTF-8?q?Chemin=20vers=20le=20sch=C3=A9ma=20de=20d?= =?UTF-8?q?onn=C3=A9es=20configurable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .template.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.template.env b/.template.env index c47e456..c397763 100644 --- a/.template.env +++ b/.template.env @@ -2,3 +2,6 @@ DATA_FILE_PARQUET_PATH=https://www.data.gouv.fr/fr/datasets/r/11cea8e8-df3e-4ed1 PORT=8050 DEVELOPMENT=True SOURCE_STATS_CSV_PATH="https://www.data.gouv.fr/api/1/datasets/r/8ded94de-3b80-4840-a5bb-7faad1c9c234" + +# Chemin vers le schéma de données +DATA_SCHEMA_PATH=https://www.data.gouv.fr/api/1/datasets/r/9a4144c0-ee44-4dec-bee5-bbef38191d9a From a44e1a37cb35ba7a9c10d99b8c938fe62b4e0134 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 17:05:37 +0200 Subject: [PATCH 30/41] =?UTF-8?q?Noms=20de=20colonnes=20friendly=20et=20d?= =?UTF-8?q?=C3=A9finition=20de=20colonne=20(tooltip)=20#33?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home.py | 60 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/src/pages/home.py b/src/pages/home.py index ede5c74..a1ebaaa 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -7,6 +7,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.utils import ( add_org_links, add_resource_link, + data_schema, filter_table_data, format_montant, format_number, @@ -73,6 +74,8 @@ datatable = html.Div( ], data_timestamp=0, markdown_options={"html": True}, + tooltip_duration=8000, + tooltip_delay=350, ), ) @@ -85,21 +88,35 @@ layout = [ ), dcc.Markdown( """ + ##### Définition des colonnes - **Filtres** + Pour voir la définition d'une colonne, passez votre souris sur son en-tête. + + ##### Filtres Vous pouvez appliquer un filtre pour chaque colonne en entrant du texte sous le nom de la colonne, puis en tapant sur `Entrée`. - - Champs textuels : la recherche est insensible à la casse (majuscules/minuscules). - - Champs numériques : vous pouvez soit taper un nombre pour trouver les valeurs égales, soit le précéder de > ou < pour filtrer les valeurs supérieures ou inférieures. + - Champs textuels : la recherche est insensible à la casse (majuscules/minuscules) et retourne les valeurs qui contiennent + le texte recherché. Exemple : `rennes` retourne "RENNES METROPOLE". + - Champs numériques : vous pouvez soit taper un nombre pour trouver les valeurs égales, soit le précéder de **>** ou **<** pour filtrer les valeurs supérieures ou inférieures. Exemple pour les offres reçues : `> 4` retourne les marchés ayant reçu plus de 4 offres. + - Champs date : vous pouvez également utiliser **>** ou **<**. Exemples : `< 2024-01-31` pour "avant le 31 janvier 2024", + `2024` pour "en 2024", `> 2022` pour "à partir de 2022" Vous pouvez filtrer plusieurs colonnes à la fois. Vos filtres sont remis à zéro quand vous rafraîchissez la page. - **Télécharger le résultat** + ##### Tri - Vous pouvez télécharger le résultat de vos filtres et tris, pour les colonnes affichées, en cliquant sur Télécharger au format Excel. + Pour trier une colonne, utilisez les flèches grises à côté des noms de colonnes. Chaque clic change le tri dans cet ordre : tri ascendant, tri descendant, pas de tri. + + ##### Télécharger le résultat + + Vous pouvez télécharger le résultat de vos filtres et tris, pour les colonnes affichées, en cliquant sur **Télécharger au format Excel**. + + ##### Liens + + Les liens dans les colonnes Acheteur et Titulaire vous permettent de consulter une vue qui leur est dédiée + (informations, marchés attribués/remportés, etc.) - Si vous téléchargez un volume important de données, il se peut que vous attendiez quelques minutes avant le début du téléchargement. """ ), ], @@ -141,7 +158,7 @@ layout = [ @callback( Output("table", "data"), Output("table", "columns"), - # Output("filtered_data", "data"), + Output("table", "tooltip_header"), Output("table", "data_timestamp"), Output("nb_rows", "children"), Output("btn-download-data", "disabled"), @@ -189,17 +206,33 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) dff = format_montant(dff) # Liste finale de colonnes - columns = [ - { - "name": column, - "id": column, + columns = [] + tooltip = {} + for column_id in dff.columns: + column_object = data_schema.get(column_id) + if column_object: + column_name = column_object.get("friendly_name", column_id) + else: + column_name = column_id + + column = { + "name": column_name, + "id": column_id, "presentation": "markdown", "type": "text", "format": {"nully": "N/A"}, "hideable": True, } - for column in dff.columns - ] + columns.append(column) + + if column_object: + tooltip[column_id] = { + "value": f"""**{column_object.get("friendly_name", column_id)}** + + """ + + column_object["description"], + "type": "markdown", + } dicts = dff.to_dicts() @@ -215,6 +248,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) return ( dicts, columns, + tooltip, data_timestamp + 1, nb_rows, download_disabled, From f01a991f4f9254e16c630b59e613c4ab6613c011 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 17:06:16 +0200 Subject: [PATCH 31/41] =?UTF-8?q?Am=C3=A9liorations=20de=20style?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/style.css | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/assets/style.css b/src/assets/style.css index a04257a..fe2f0a0 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -18,6 +18,14 @@ margin-bottom: 25px; } +#instructions { + max-width: 1000px; +} + +details > div { + padding-top: 24px; +} + /* Réduire la taille du texte de la colonne Objet */ td[data-dash-column="objet"] { @@ -31,6 +39,7 @@ td[data-dash-column="objet"] { th.dash-header { background-color: #b33821; color: white; + font-family: sans-serif; } .dash-table-container @@ -48,10 +57,33 @@ td[data-dash-column="objet"] { display: none; } +.dash-tooltip, +.dash-table-tooltip { + color: #333; + width: 400px !important; + max-width: 400px !important; + height: 100px !important; + max-height: 100px !important; + overflow: hidden; +} + +.dash-tooltip pre, +.dash-tooltip code { + overflow: hidden; + height: 150px; + text-wrap: wrap; + font-family: sans-serif; +} + /* Menu de masquage des colonnes */ -.column-header--hide svg { +.column-actions { + margin-right: 8px; +} + +.column-header--hide { display: none; } + .show-hide { position: relative; width: 180px; From ad6005259e56bfbe022973f694d576ac42daa24b Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 17:07:35 +0200 Subject: [PATCH 32/41] Les filtres <> marchent aussi avec les strings #42 --- src/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index d49a7f5..64e1244 100644 --- a/src/utils.py +++ b/src/utils.py @@ -216,7 +216,9 @@ def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: lff = dates_to_strings(lff, col_name) if operator in ("<", "<=", ">", ">="): - filter_value = int(filter_value) + lff = lff.filter( + pl.col(col_name).is_not_null() & (pl.col(col_name) != pl.lit("")) + ) if operator == "<": lff = lff.filter(pl.col(col_name) < filter_value) elif operator == ">": From ee2bae190d3ce986d8a34d01e9a6de35f377ffa1 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 17:08:45 +0200 Subject: [PATCH 33/41] =?UTF-8?q?R=C3=A9cup=C3=A9ration=20du=20sch=C3=A9ma?= =?UTF-8?q?=20#33?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/utils.py b/src/utils.py index 64e1244..91e5d0b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -268,3 +268,6 @@ meta_content = { "Pour une commande publique accessible à toutes et tous." ), } + +# Récupération du schéma des données tabulaires +data_schema: dict = get(os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True).json() From 1d21cbb69d4d39e21d5d8afd6ecfd586495a852e Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 17:36:03 +0200 Subject: [PATCH 34/41] =?UTF-8?q?D=C3=A9but=20de=20configuration=20de=20co?= =?UTF-8?q?lonnes=20partag=C3=A9e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/style.css | 1 + src/pages/acheteur.py | 25 ++++++++++--------------- src/pages/home.py | 31 ++----------------------------- src/pages/titulaire.py | 21 ++++++++------------- src/utils.py | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 57 deletions(-) diff --git a/src/assets/style.css b/src/assets/style.css index fe2f0a0..00f20a0 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -40,6 +40,7 @@ td[data-dash-column="objet"] { background-color: #b33821; color: white; font-family: sans-serif; + text-align: left; } .dash-table-container diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 54c3ada..6af4f51 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -12,6 +12,7 @@ from src.utils import ( get_departement_region, lf, meta_content, + setup_table_columns, ) register_page( @@ -24,8 +25,6 @@ register_page( order=5, ) -# 21690123100011 - layout = [ dcc.Store(id="acheteur_data", storage_type="memory"), dcc.Location(id="url", refresh="callback-nav"), @@ -214,30 +213,26 @@ def get_last_marches_table(data) -> html.Div: dff = pl.DataFrame(data) dff = format_montant(dff) + columns, tooltip = setup_table_columns( + dff, hideable=False, exclude=["titulaire_id", "titulaire_typeIdentifiant"] + ) data = dff.to_dicts() data = add_org_links_in_dict(data, "titulaire") table = html.Div( className="marches_table", - id="marches_datatable", + id="acheteur_datatable", children=dash_table.DataTable( data=data, markdown_options={"html": True}, page_action="native", filter_action="native", filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": False, - } - for i in columns - if i not in ["titulaire_id", "titulaire_typeIdentifiant"] - ], + columns=columns, + tooltip_header=tooltip, + tooltip_duration=8000, + tooltip_delay=350, + cell_selectable=False, page_size=10, style_cell_conditional=[ { diff --git a/src/pages/home.py b/src/pages/home.py index a1ebaaa..b10465a 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -7,12 +7,12 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.utils import ( add_org_links, add_resource_link, - data_schema, filter_table_data, format_montant, format_number, lf, meta_content, + setup_table_columns, sort_table_data, ) @@ -205,34 +205,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) # Formatage des montants dff = format_montant(dff) - # Liste finale de colonnes - columns = [] - tooltip = {} - for column_id in dff.columns: - column_object = data_schema.get(column_id) - if column_object: - column_name = column_object.get("friendly_name", column_id) - else: - column_name = column_id - - column = { - "name": column_name, - "id": column_id, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": True, - } - columns.append(column) - - if column_object: - tooltip[column_id] = { - "value": f"""**{column_object.get("friendly_name", column_id)}** - - """ - + column_object["description"], - "type": "markdown", - } + columns, tooltip = setup_table_columns(dff) dicts = dff.to_dicts() diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 7ea1a2a..2816f0f 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -12,6 +12,7 @@ from src.utils import ( get_departement_region, lf, meta_content, + setup_table_columns, ) register_page( @@ -216,6 +217,7 @@ def get_last_marches_table(data) -> html.Div: dff = pl.DataFrame(data) dff = format_montant(dff) + columns, tooltip = setup_table_columns(dff, hideable=False, exclude=["acheteur_id"]) data = dff.to_dicts() # Idéalement on utiliserait add_org_links(), mais le résultat attendu # est différent de home.py (Tableau) @@ -223,25 +225,18 @@ def get_last_marches_table(data) -> html.Div: table = html.Div( className="marches_table", - id="marches_datatable", + id="titulaire_datatable", children=dash_table.DataTable( data=data, markdown_options={"html": True}, page_action="native", filter_action="native", filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": False, - } - for i in columns - if i not in ["titulaire_id"] - ], + columns=columns, + tooltip_header=tooltip, + tooltip_duration=8000, + tooltip_delay=350, + cell_selectable=False, page_size=10, style_cell_conditional=[ { diff --git a/src/utils.py b/src/utils.py index 91e5d0b..dc55c19 100644 --- a/src/utils.py +++ b/src/utils.py @@ -255,6 +255,40 @@ def sort_table_data(lff: pl.LazyFrame, sort_by: list) -> pl.LazyFrame: return lff +def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tuple: + # Liste finale de colonnes + columns = [] + tooltip = {} + for column_id in dff.columns: + if exclude and column_id in exclude: + continue + column_object = data_schema.get(column_id) + if column_object: + column_name = column_object.get("friendly_name", column_id) + else: + column_name = column_id + + column = { + "name": column_name, + "id": column_id, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": hideable, + } + columns.append(column) + + if column_object: + tooltip[column_id] = { + "value": f"""**{column_object.get("friendly_name", column_id)}** + + """ + + column_object["description"], + "type": "markdown", + } + return columns, tooltip + + lf = get_decp_data() departements = get_departements() domain_name = ( From 928f75f5045ce934816103586a44cb959e84a2b3 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 19:32:15 +0200 Subject: [PATCH 35/41] Formulaire de contact avec rate limiting simple #48 --- .template.env | 7 +++++++ README.md | 9 ++++++--- src/app.py | 11 +---------- src/pages/about.py | 10 +--------- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.template.env b/.template.env index c397763..7b600ee 100644 --- a/.template.env +++ b/.template.env @@ -5,3 +5,10 @@ SOURCE_STATS_CSV_PATH="https://www.data.gouv.fr/api/1/datasets/r/8ded94de-3b80-4 # Chemin vers le schéma de données DATA_SCHEMA_PATH=https://www.data.gouv.fr/api/1/datasets/r/9a4144c0-ee44-4dec-bee5-bbef38191d9a + +# Formulaire de contact +SENDER_SERVER_DOMAIN="mail.example.com" # serveur SMTP +LOGIN_PASSWORD="" # mot de passe du serveur +LOGIN_EMAIL="connect@example.fr" # adresse utilisée pour se connecter au serveur SMTP +FROM_EMAIL="colin+contactform@maudry.com" # dresse d'envoi des emails (From) +TO_EMAIL="colin+decp@maudry.com" # adresse de destination des emails (To) diff --git a/README.md b/README.md index e87977d..4428ef8 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,12 @@ python run.py #### 2.1.0 -- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) et [titulaire](https://decp.info/titulaires/51903758414786) ([#28](https://github.com/ColinMaudry/decp.info/issues/28) et [#35](https://github.com/ColinMaudry/decp.info/issues/35)) -- Ajout des balises HTML meta Open Graph et Twitter ([#39](https://github.com/ColinMaudry/decp.info/issues/39)) pour de beaux aperçus de liens -- Variables globales uniquement en lecture +- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) et [titulaire](https://decp.info/titulaires/51903758414786) ([#28](https://github.com/ColinMaudry/decp.info/issues/28) et [#35](https://github.com/ColinMaudry/decp.info/issues/35)) 🔎 +- Ajout des balises HTML meta Open Graph et Twitter ([#39](https://github.com/ColinMaudry/decp.info/issues/39)) pour de beaux aperçus de liens 🖼️ +- Formulaire de contact ([#48](https://github.com/ColinMaudry/decp.info/issues/48)) 📨 +- Nom de colonnes plus_agréables ([#33](https://github.com/ColinMaudry/decp.info/issues/33)) 💅 +- Définition des colonnes quand vous passez votre souris sur les en-têtes ([#33](https://github.com/ColinMaudry/decp.info/issues/33)) 📖 +- Variables globales uniquement en lecture (😁) ##### 2.0.1 (23 septembre 2025) diff --git a/src/app.py b/src/app.py index da958c6..c8ecc2f 100644 --- a/src/app.py +++ b/src/app.py @@ -77,7 +77,7 @@ app.layout = html.Div( page["name"], href=page["relative_path"], className="nav" ) for page in page_registry.values() - if page["name"] not in ["Acheteur", "Fournisseur"] + if page["name"] not in ["Acheteur", "Titulaire"] ] ), ], @@ -86,15 +86,6 @@ app.layout = html.Div( page_container, ] ) -# @callback( -# Output(component_id="table", component_property="data", allow_duplicate=True), -# Input(component_id="search", component_property="value"), -# prevent_initial_call=True, -# ) -# def global_search(text): -# new_df = df -# new_df = new_df.filter(pl.col("objet").str.contains("(?i)" + text)) -# return new_df.to_dicts() if __name__ == "__main__": app.run(debug=True) diff --git a/src/pages/about.py b/src/pages/about.py index b44a7ee..d4b60aa 100644 --- a/src/pages/about.py +++ b/src/pages/about.py @@ -42,7 +42,7 @@ les fonctionnalités actuelles de decp.info. Il est ainsi possible de rajouter dcc.Markdown(""" - via l'achat d'une prestation de service (devis, prestation, facture), vous pouvez financer le développement de [fonctionnalités prévues](https://github.com/ColinMaudry/decp.info/issues), ou d'autres ! - ma société accepte aussi les dons (pas de réduction d'impôt possible) -- envoyez un mail et on discute ! +- [écrivez-moi](/contact) et on discute ! #### Pour explorer le projet @@ -54,14 +54,6 @@ les fonctionnalités actuelles de decp.info. Il est ainsi possible de rajouter - [de decp.info](https://github.com/ColinMaudry/decp.info) - [du traitement des données](https://github.com/ColinMaudry/decp-processing) """), - html.H4("Contact", id="contact"), - dcc.Markdown(""" -- Email : [colin+decp@maudry.com](mailto:colin+decp@maudry.com) -- Bluesky : [@col1m.bsky.social](https://bsky.app/profile/col1m.bsky.social) -- Mastodon : [col1m@mamot.fr](https://mamot.fr/@col1m) -- LinkedIn : [colinmaudry](https://www.linkedin.com/in/colinmaudry/) -- venez discuter de la transparence de la commande publique [sur le forum teamopendata.org](https://teamopendata.org/c/commande-publique/101) -"""), html.H4("Sources de données", id="sources"), get_sources_tables(os.getenv("SOURCE_STATS_CSV_PATH")), html.H4("Mentions légales", id="mentions-legales"), From 819f2e315f702a2ce025e51d559d30fa6cf43a11 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 29 Sep 2025 20:06:04 +0200 Subject: [PATCH 36/41] =?UTF-8?q?Affichage=20du=20num=C3=A9ro=20de=20versi?= =?UTF-8?q?on=20et=20lien=20vers=20changelog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + pyproject.toml | 2 +- src/app.py | 21 ++++++++++++++++++++- src/assets/style.css | 30 +++++++++++++++++++++++++----- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4428ef8..ee53ae6 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ python run.py - Formulaire de contact ([#48](https://github.com/ColinMaudry/decp.info/issues/48)) 📨 - Nom de colonnes plus_agréables ([#33](https://github.com/ColinMaudry/decp.info/issues/33)) 💅 - Définition des colonnes quand vous passez votre souris sur les en-têtes ([#33](https://github.com/ColinMaudry/decp.info/issues/33)) 📖 +- Affichage du numéro de version près du logo et lien vers ici 🤓 - Variables globales uniquement en lecture (😁) ##### 2.0.1 (23 septembre 2025) diff --git a/pyproject.toml b/pyproject.toml index e533afa..9b7f6ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "decp.info" description = "" -version = "2.0.1" +version = "2.1.0" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin+decp@maudry.com" } diff --git a/src/app.py b/src/app.py index c8ecc2f..32b659a 100644 --- a/src/app.py +++ b/src/app.py @@ -1,6 +1,7 @@ import logging import dash_bootstrap_components as dbc +import tomllib from dash import Dash, dcc, html, page_container, page_registry from dotenv import load_dotenv from flask import send_from_directory @@ -32,6 +33,11 @@ logging.basicConfig( datefmt="%Y-%m-%d %H:%M:%S", ) +with open("./pyproject.toml", "rb") as f: + pyproject = tomllib.load(f) + version = "v" + pyproject["project"]["version"] + + app.index_string = """ @@ -70,7 +76,20 @@ app.layout = html.Div( [ html.Div( [ - html.H1("decp.info"), + html.Div( + [ + html.A(children=html.H1("decp.info"), href="/"), + html.P( + children=html.A( + version, + href="https://github.com/ColinMaudry/decp.info?tab=readme-ov-file#notes-de-version", + target="_blank", + ), + className="version", + ), + ], + className="logo", + ), html.Div( [ dcc.Link( diff --git a/src/assets/style.css b/src/assets/style.css index 00f20a0..66f426a 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -26,6 +26,31 @@ details > div { padding-top: 24px; } +/* Logo et version */ + +div.logo { + width: 230px; +} + +p.version { + float: left; + margin-top: 21px; + margin-left: 12px; +} + +p.version > a { + text-decoration: none; +} + +div.logo h1 { + float: left; +} + +div.logo > a { + text-decoration: none; + color: black; +} + /* Réduire la taille du texte de la colonne Objet */ td[data-dash-column="objet"] { @@ -133,11 +158,6 @@ a.nav { font-size: 120%; } -.navbar.h1 { - float: left; - width: 50%; -} - h3 { margin: 36px 0 24px 0; } From 79d11b326ee1610cc5da1e55839ceaa1d11acbd9 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 30 Sep 2025 11:35:18 +0200 Subject: [PATCH 37/41] Formulaire de contact avec rate limiting simple #48 --- src/pages/contact.py | 170 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 src/pages/contact.py diff --git a/src/pages/contact.py b/src/pages/contact.py new file mode 100644 index 0000000..a6f4051 --- /dev/null +++ b/src/pages/contact.py @@ -0,0 +1,170 @@ +import os +import re +import smtplib +import time +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from dash import Input, Output, State, callback, dcc, html, register_page +from flask import request + +from src.utils import meta_content + +name = "Contact" +register_page( + __name__, + path="/contact", + title=meta_content["title"], + name=name, + description=meta_content["description"], + image_url=meta_content["image_url"], + order=6, +) + +layout = html.Div( + className="container", + children=[ + html.H2("Contact", id="contact"), + html.P( + "Votre message arrivera directement dans ma boîte mail, et je reviendrai vers vous rapidement." + ), + html.Div( + [ + html.Label("Votre nom"), + dcc.Input( + id="input-name", + type="text", + style={"width": "100%", "padding": "8px", "margin": "8px 0"}, + ), + html.Label("Votre adresse email"), + dcc.Input( + id="input-email", + type="email", + style={"width": "100%", "padding": "8px", "margin": "8px 0"}, + ), + html.Label("Message"), + dcc.Textarea( + id="input-message", + style={ + "width": "100%", + "height": 200, + "padding": "8px", + "margin": "8px 0", + }, + ), + html.Button( + "Envoyer ", + id="submit-button", + n_clicks=0, + style={"marginTop": "10px"}, + ), + html.Div( + id="output-message", style={"marginTop": "10px", "color": "green"} + ), + ], + style={ + "maxWidth": "600px", + "margin": "auto", + "padding": "20px", + "lineHeight": "20px", + }, + ), + dcc.Markdown(""" +- Bluesky : [@col1m.bsky.social](https://bsky.app/profile/col1m.bsky.social) +- Mastodon : [col1m@mamot.fr](https://mamot.fr/@col1m) +- LinkedIn : [colinmaudry](https://www.linkedin.com/in/colinmaudry/) +- venez discuter de la transparence de la commande publique [sur le forum teamopendata.org](https://teamopendata.org/c/commande-publique/101) +"""), + ], +) + +rate_limit_store = {} # {ip: last_timestamp} +RATE_LIMIT_WINDOW = 300 # 5 minutes + + +def is_rate_limited(): + ip = request.remote_addr + now = time.time() + last_time = rate_limit_store.get(ip) + + if last_time and (now - last_time) < RATE_LIMIT_WINDOW: + return True # rate limited ! + rate_limit_store[ip] = now # màj du timestamp + return False + + +def sanitize_email(email): + return re.match(r"[^@]+@[^@]+\.[^@]+", email) + + +@callback( + Output("output-message", "children"), + Input("submit-button", "n_clicks"), + State("input-name", "value"), + State("input-email", "value"), + State("input-message", "value"), + prevent_initial_call=True, +) +def send_email(n_clicks, form_name, form_email, form_message): + if not all([form_name, form_email, form_message]): + return html.Div( + "Veuillez s'il vous plaît remplir tous les champs.", style={"color": "red"} + ) + + client_ip = request.remote_addr + if is_rate_limited(): + wait_time = int(RATE_LIMIT_WINDOW - (time.time() - rate_limit_store[client_ip])) + return html.Div( + f"⏳ J'ai mis en place une protection contre le spam, et vous m'avez écrit il y a moins de 5 minutes. " + f"Veuillez s'il vous plaît attendre {wait_time} secondes avant de renvoyer un message.", + style={"color": "black"}, + ) + + try: + # Configuration du serveur SMTP + smtp_server = os.getenv("SENDER_SERVER_DOMAIN") + smtp_port = 587 + login_email = os.getenv("LOGIN_EMAIL") + from_email = os.getenv("FROM_EMAIL") + to_email = os.getenv("TO_EMAIL", from_email) + login_password = os.getenv("LOGIN_PASSWORD") + + print( + smtp_server, + smtp_port, + login_email, + from_email, + to_email, + login_password, + sep="\n", + ) + + # Création de l'email + email = MIMEMultipart() + email["From"] = from_email + email["To"] = to_email + email["Subject"] = f"[decp.info] Message de {form_name}" + + body = f""" + Nom : {form_name} + Email : {form_email} + + Message : + {form_message} + """ + email.attach(MIMEText(body, "plain")) + + # Send email + server = smtplib.SMTP(smtp_server, smtp_port) + server.starttls() + server.login(login_email, login_password) + server.sendmail(from_email, to_email, email.as_string()) + server.quit() + + return html.Div("✅ Envoi réussi", style={"color": "green"}) + + except Exception as e: + print(e) + return html.Div( + f"❌ Échec de l'envoi du message : {str(e)}", style={"color": "red"} + ) From 7c157d9d71bc7fb639efef39d3b05c9de0bb61ac Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 30 Sep 2025 14:57:41 +0200 Subject: [PATCH 38/41] =?UTF-8?q?Vue=20march=C3=A9=20simple=20#40?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- src/app.py | 2 +- src/assets/style.css | 35 +++++++- src/pages/about.py | 2 +- src/pages/acheteur.py | 19 ++--- src/pages/home.py | 4 +- src/pages/marche.py | 188 +++++++++++++++++++++++++++++++++++++++++ src/pages/titulaire.py | 13 +-- src/utils.py | 32 ++++--- 9 files changed, 260 insertions(+), 37 deletions(-) create mode 100644 src/pages/marche.py diff --git a/README.md b/README.md index ee53ae6..d436933 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ python run.py #### 2.1.0 -- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) et [titulaire](https://decp.info/titulaires/51903758414786) ([#28](https://github.com/ColinMaudry/decp.info/issues/28) et [#35](https://github.com/ColinMaudry/decp.info/issues/35)) 🔎 +- Ajout des vues [acheteur](https://decp.info/acheteurs/24350013900189) ([#28](https://github.com/ColinMaudry/decp.info/issues/28)), [titulaire](https://decp.info/titulaires/51903758414786) ([#35](https://github.com/ColinMaudry/decp.info/issues/35)) et [marché](https://decp.info/marches/532239472000482025S00004) ([#40](https://github.com/ColinMaudry/decp.info/issues/40)) 🔎 - Ajout des balises HTML meta Open Graph et Twitter ([#39](https://github.com/ColinMaudry/decp.info/issues/39)) pour de beaux aperçus de liens 🖼️ - Formulaire de contact ([#48](https://github.com/ColinMaudry/decp.info/issues/48)) 📨 - Nom de colonnes plus_agréables ([#33](https://github.com/ColinMaudry/decp.info/issues/33)) 💅 diff --git a/src/app.py b/src/app.py index 32b659a..71fdb43 100644 --- a/src/app.py +++ b/src/app.py @@ -96,7 +96,7 @@ app.layout = html.Div( page["name"], href=page["relative_path"], className="nav" ) for page in page_registry.values() - if page["name"] not in ["Acheteur", "Titulaire"] + if page["name"] not in ["Acheteur", "Titulaire", "Marché"] ] ), ], diff --git a/src/assets/style.css b/src/assets/style.css index 66f426a..bf70592 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -88,8 +88,8 @@ td[data-dash-column="objet"] { color: #333; width: 400px !important; max-width: 400px !important; - height: 100px !important; - max-height: 100px !important; + height: 150px !important; + max-height: 150px !important; overflow: hidden; } @@ -179,6 +179,9 @@ summary > h3 { justify-content: space-between; } +.wrapper > div { +} + .org_title { grid-column: 1 / 3; grid-row: 1; @@ -207,3 +210,31 @@ summary > h3 { grid-column: 3; grid-row: 2; } + +/* Vue marché */ + +.marche_infos p { + margin-top: 6px; + margin-bottom: 6px; +} + +.marche_title { + grid-column: 1/3; + grid-row: 1; +} + +.marche_map { + grid-column: 3; + grid-row: 2; + width: 400px; +} + +.marche_infos_1 { + grid-column: 1; + grid-row: 2; +} + +.marche_infos_2 { + grid-column: 2; + grid-row: 2; +} diff --git a/src/pages/about.py b/src/pages/about.py index d4b60aa..f2278e3 100644 --- a/src/pages/about.py +++ b/src/pages/about.py @@ -48,7 +48,7 @@ les fonctionnalités actuelles de decp.info. Il est ainsi possible de rajouter - ✉️ [inscription à la liste de diffusion](https://6254d9a3.sibforms.com/serve/MUIFAEonUVkoSVrdgey18CTgLyI16xw4yeu-M-YOUzhWE_AgfQfbgkyT7GvA_RYLro9MfuRqkzQxSvu7-uzbMSv2a2ZQPsliM7wtiiqIL8kR2zOvl6m11fb5qjcOxMAYsLiY_YBi3P7NY95CTJ8vRY4CpsDclF2iLooOElKkTgIgi5nePe7zAIrgiYM5v2EuALlGJZMEG9vBP-Cu) (annonces des mises à jour et évènements, maximum une fois par mois) - 💾 [données consolidées en Open Data](https://www.data.gouv.fr/datasets/donnees-essentielles-de-la-commande-publique-consolidees-format-tabulaire/) -- 🗞️ [mon blog](https://colin.maudry.com), qui parle beaucoup de transparence des marchés publics +- 🗞️ [mon blog](https://colin.maudry.com) - 📔 [wiki du projet](https://github.com/ColinMaudry/decp-processing/wiki) - 🚰 code source - [de decp.info](https://github.com/ColinMaudry/decp.info) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 6af4f51..4245b00 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -5,7 +5,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.figures import point_on_map from src.utils import ( - add_org_links_in_dict, + add_links_in_dict, format_montant, format_number, get_annuaire_data, @@ -177,6 +177,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: lff = lff.fill_null("") lff = lff.select( "id", + "uid", "objet", "dateNotification", "titulaire_id", @@ -201,23 +202,15 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: Input(component_id="acheteur_data", component_property="data"), ) def get_last_marches_table(data) -> html.Div: - columns = [ - "id", - "objet", - "dateNotification", - "titulaire_nom", - "montant", - "codeCPV", - "dureeMois", - ] - dff = pl.DataFrame(data) dff = format_montant(dff) columns, tooltip = setup_table_columns( - dff, hideable=False, exclude=["titulaire_id", "titulaire_typeIdentifiant"] + dff, + hideable=False, + exclude=["titulaire_id", "titulaire_typeIdentifiant", "uid"], ) data = dff.to_dicts() - data = add_org_links_in_dict(data, "titulaire") + data = add_links_in_dict(data, "titulaire") table = html.Div( className="marches_table", diff --git a/src/pages/home.py b/src/pages/home.py index b10465a..8d173c3 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -5,7 +5,7 @@ import polars as pl from dash import Input, Output, State, callback, dash_table, dcc, html, register_page from src.utils import ( - add_org_links, + add_links, add_resource_link, filter_table_data, format_montant, @@ -197,7 +197,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) dff = dff.slice(start_row, page_size) # Ajout des liens vers l'annuaire des entreprises - dff = add_org_links(dff) + dff = add_links(dff) # Ajout des liens vers les fichiers Open Data dff = add_resource_link(dff) diff --git a/src/pages/marche.py b/src/pages/marche.py new file mode 100644 index 0000000..88c1baf --- /dev/null +++ b/src/pages/marche.py @@ -0,0 +1,188 @@ +from datetime import datetime + +import polars as pl +from dash import Input, Output, callback, dcc, html, register_page +from polars import selectors as cs + +from src.utils import data_schema, format_montant, lf, meta_content + +register_page( + __name__, + path_template="/marches/", + title=meta_content["title"], + name="Marché", + description=meta_content["description"], + image_url=meta_content["image_url"], + order=7, +) + +layout = [ + dcc.Store(id="marche_data"), + dcc.Store(id="titulaires_data"), + dcc.Location(id="url", refresh="callback-nav"), + html.Div( + className="container marche_infos", + children=[ + html.P("Vous consultez un résumé des données de ce marché public"), + html.Ul( + [ + html.Li( + "après son attribution aux titulaires qui l'ont remporté à la suite d'un appel d'offres (ou sans appel d'offres via une attribution directe)" + ), + html.Li( + "après avoir appliqué les éventuelles modifications de montant, durée ou titulaires renseignées par l'acheteur" + ), + ] + ), + html.P( + "Le montant total payé aux titulaires, la durée du marché et la liste des titulaires peuvent cependant encore évoluer jusqu'à la fin de l'exécution du marché." + ), + html.Div( + className="wrapper", + children=[ + html.Div( + className="marche_map", + id="marche_map", + children=[ + html.H4("Titulaires"), + html.Ul(id="marche_infos_titulaires"), + ], + ), + html.Div(className="marche_infos_1", id="marche_infos_1"), + html.Div(className="marche_infos_2", id="marche_infos_2"), + ], + ), + ], + ), +] + + +@callback( + Output("marche_data", "data"), + Output("titulaires_data", "data"), + Input(component_id="url", component_property="pathname"), +) +def get_marche_data(url): + marche_uid = url.split("/")[-1] + + # Récupération des données du marché à partir du lf global + lff = lf.filter(pl.col("uid") == pl.lit(marche_uid)) + + # Données des titulaires du marché + dff_titulaires = lff.select(cs.starts_with("titulaire")).collect(engine="streaming") + + # Données du marché + dff_marche = ( + lff.select(~cs.starts_with("titulaires")).unique().collect(engine="streaming") + ) + dff_marche = format_montant(dff_marche) + + assert dff_marche.height == 1 + + return dff_marche.to_dicts()[0], dff_titulaires.to_dicts() + + +@callback( + Output("marche_infos_1", "children"), + Output("marche_infos_2", "children"), + Output("marche_infos_titulaires", "children"), + Input("marche_data", "data"), + Input("titulaires_data", "data"), +) +def update_marche_info(marche, titulaires): + def make_parameter(col): + column_object = data_schema.get(col) + column_name = column_object.get("friendly_name") if column_object else col + + if marche[col]: + if col == "acheteur_nom": + value = html.A( + href=f"/acheteurs/{marche['acheteur_id']}", + children=marche["acheteur_nom"], + ) + elif col == "sourceDataset": + value = html.A( + href=marche["sourceFile"], children=marche["sourceDataset"] + ) + column_name = "Source des données" + + # Dates + elif col in ["dateNotification", "datePublicationDonnees"]: + print(marche[col]) + + value = datetime.fromisoformat(marche[col]).strftime("%d/%m/%Y") + + # Listes + elif ( + col + in [ + "techniques", + "typesPrix", + "considerationsSociales", + "considerationsEnvironnementales", + ] + and "," in marche[col] + ): + col_values = marche[col].split(", ") + lines = [] + for val in col_values: + lines.append(html.Li(val)) + _content = html.Div( + [html.P([column_name, " : "]), html.Ul(children=lines)] + ) + return _content + else: + value = marche.get(col) + else: + value = "" + + param_content = html.P([column_name, " : ", html.Strong(value)]) + return param_content + + marche_infos = [ + make_parameter("id"), + make_parameter("objet"), + make_parameter("dateNotification"), # date + make_parameter("nature"), + make_parameter("acheteur_nom"), # lien + make_parameter("montant"), + make_parameter("codeCPV"), + make_parameter("procedure"), + make_parameter("techniques"), # list + make_parameter("dureeMois"), + make_parameter("offresRecues"), + make_parameter("datePublicationDonnees"), # date + make_parameter("formePrix"), + make_parameter("typesPrix"), # list + make_parameter("attributionAvance"), + make_parameter("tauxAvance"), + make_parameter("marcheInnovant"), # label + make_parameter("modalitesExecution"), + make_parameter("considerationsSociales"), # list + make_parameter("considerationsEnvironnementales"), # list + make_parameter("ccag"), + make_parameter("sousTraitanceDeclaree"), + make_parameter("typeGroupementOperateurs"), + make_parameter("origineFrance"), + make_parameter("origineUE"), + make_parameter("idAccordCadre"), + make_parameter("sourceDataset"), # lien + ] + + half = round(len(marche_infos) / 2) + # pas inclus pour l'instant : lieu d'exécution, modifications + + titulaires_lines = [] + for titulaire in titulaires: + if titulaire["titulaire_typeIdentifiant"] == "SIRET": + content = html.Li( + html.A( + href=f"/titulaires/{titulaire['titulaire_id']}", + children=titulaire["titulaire_nom"], + ) + ) + else: + content = html.Li(titulaire["titulaire_nom"]) + titulaires_lines.append(content) + + return marche_infos[:half], marche_infos[half:], titulaires_lines diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 2816f0f..bca19c4 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -5,7 +5,7 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.figures import point_on_map from src.utils import ( - add_org_links_in_dict, + add_links_in_dict, format_montant, format_number, get_annuaire_data, @@ -181,6 +181,7 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> pl.LazyFrame: ) lff = lff.fill_null("") lff = lff.select( + "id", "uid", "objet", "dateNotification", @@ -217,11 +218,13 @@ def get_last_marches_table(data) -> html.Div: dff = pl.DataFrame(data) dff = format_montant(dff) - columns, tooltip = setup_table_columns(dff, hideable=False, exclude=["acheteur_id"]) + columns, tooltip = setup_table_columns( + dff, hideable=False, exclude=["acheteur_id", "id"] + ) data = dff.to_dicts() # Idéalement on utiliserait add_org_links(), mais le résultat attendu # est différent de home.py (Tableau) - data = add_org_links_in_dict(data, "acheteur") + data = add_links_in_dict(data, "acheteur") table = html.Div( className="marches_table", @@ -248,8 +251,8 @@ def get_last_marches_table(data) -> html.Div: "whiteSpace": "normal", }, { - "if": {"column_id": " acheteur_nom"}, - "minWidth": "200px", + "if": {"column_id": "acheteur_nom"}, + "maxWidth": "400px", "textAlign": "left", "overflow": "hidden", "lineHeight": "18px", diff --git a/src/utils.py b/src/utils.py index dc55c19..4aaa579 100644 --- a/src/utils.py +++ b/src/utils.py @@ -49,7 +49,7 @@ def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame: return dff -def add_org_links(dff: pl.DataFrame): +def add_links(dff: pl.DataFrame): dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( @@ -62,25 +62,29 @@ def add_org_links(dff: pl.DataFrame): .otherwise(pl.col("titulaire_id")) .alias("titulaire_id") ) - dff = dff.with_columns( - ( - '' - + pl.col("acheteur_id") - + "" - ).alias("acheteur_id") - ) + + for column, path in [("acheteur_id", "acheteurs"), ("uid", "marches")]: + dff = dff.with_columns( + ( + f'' + + pl.col(column) + + "" + ).alias(column) + ) return dff -def add_org_links_in_dict(data: list, org_type: str) -> list: +def add_links_in_dict(data: list, org_type: str) -> list: new_data = [] for marche in data: org_id = marche[org_type + "_id"] marche[org_type + "_nom"] = ( f'{marche[org_type + "_nom"]}' ) + marche["id"] = f'{marche["id"]}' + marche["uid"] = f'{marche["uid"]}' new_data.append(marche) return new_data @@ -280,7 +284,7 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup if column_object: tooltip[column_id] = { - "value": f"""**{column_object.get("friendly_name", column_id)}** + "value": f"""**{column_object.get("friendly_name")}** ({column_id}) """ + column_object["description"], @@ -305,3 +309,7 @@ meta_content = { # Récupération du schéma des données tabulaires data_schema: dict = get(os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True).json() +data_schema["source"] = { + "description": "Code de la source des données, avec un lien vers le fichier Open Data dont proviennent les données de ce marché public.", + "friendly_name": "Source des données", +} From bc930bf6c9c7853c77f69d58b165bd49f88b157e Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 30 Sep 2025 15:09:21 +0200 Subject: [PATCH 39/41] =?UTF-8?q?M=C3=A0j=20instruction=20#40?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/home.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/home.py b/src/pages/home.py index 8d173c3..e309bc0 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -114,7 +114,7 @@ layout = [ ##### Liens - Les liens dans les colonnes Acheteur et Titulaire vous permettent de consulter une vue qui leur est dédiée + Les liens dans les colonnes Identifiant unique, Acheteur et Titulaire vous permettent de consulter une vue qui leur est dédiée (informations, marchés attribués/remportés, etc.) """ From 0b828c3d677327a0349238cdef0dcd2d489a3a51 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 30 Sep 2025 17:12:41 +0200 Subject: [PATCH 40/41] =?UTF-8?q?Num=C3=A9ro=20de=20version,=20description?= =?UTF-8?q?=20toml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d436933..0b03ed6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # decp.info -> v2.0.1 +> v2.1.0 Outil d'exploration et de téléchargement des données essentielles de la commande publique. diff --git a/pyproject.toml b/pyproject.toml index 9b7f6ff..acfbe79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "decp.info" -description = "" +description = "Interface d'exploration et d'analyse des marchés publics français." version = "2.1.0" requires-python = ">= 3.10" authors = [ From e4dcc19e8b5f5acb78d834de23b93c67ba88b3a1 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 30 Sep 2025 17:13:42 +0200 Subject: [PATCH 41/41] template.env --- .template.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.template.env b/.template.env index 7b600ee..0eec464 100644 --- a/.template.env +++ b/.template.env @@ -10,5 +10,5 @@ DATA_SCHEMA_PATH=https://www.data.gouv.fr/api/1/datasets/r/9a4144c0-ee44-4dec-be SENDER_SERVER_DOMAIN="mail.example.com" # serveur SMTP LOGIN_PASSWORD="" # mot de passe du serveur LOGIN_EMAIL="connect@example.fr" # adresse utilisée pour se connecter au serveur SMTP -FROM_EMAIL="colin+contactform@maudry.com" # dresse d'envoi des emails (From) -TO_EMAIL="colin+decp@maudry.com" # adresse de destination des emails (To) +FROM_EMAIL="from@example.com" # adresse d'envoi des emails (From) +TO_EMAIL="to@example.com" # adresse de destination des emails (To)