From c7d1a5ec731847f0aecd0aa2331d819e5d25958d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 13 Mar 2026 19:22:48 +0100 Subject: [PATCH 01/75] =?UTF-8?q?D=C3=A9but=20de=20dashboard=20avec=20quel?= =?UTF-8?q?ques=20filtres=20et=20viz=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 4 + src/figures.py | 132 +++++++++++++++++++++++-- src/pages/statistiques.py | 201 ++++++++++++++++++++++++++++---------- src/utils.py | 10 ++ 4 files changed, 290 insertions(+), 57 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index e76e1f2..a183642 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -151,6 +151,10 @@ p.version > a { max-width: 900px; } +.seeBorder { + border: dotted 1px green; +} + /* --- Search Page --- */ .tagline { text-align: center; diff --git a/src/figures.py b/src/figures.py index e95b779..f06ae37 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,6 +1,8 @@ import json from typing import Literal +from urllib.error import HTTPError, URLError +import dash_bootstrap_components as dbc import plotly.express as px import plotly.graph_objects as go import polars as pl @@ -161,8 +163,11 @@ def get_barchart_sources(df_source: pl.DataFrame, type_date: str): def get_sources_tables(source_path) -> html.Div: - df = pl.read_csv(source_path) - df = df.with_columns( + try: + dff = pl.read_csv(source_path) + except (URLError, HTTPError): + return html.Div("Erreur de connexion") + dff = dff.with_columns( ( pl.lit(' html.Div: + pl.lit("") ).alias("nom") ) - df = df.drop("url", "unique") - df = df.sort(by=["nb_marchés"], descending=True) + dff = dff.drop("url", "unique") + dff = dff.sort(by=["nb_marchés"], descending=True) columns = { "nom": "Nom de la source", @@ -184,7 +189,7 @@ def get_sources_tables(source_path) -> html.Div: datatable = dash_table.DataTable( id="source_table", - data=df.to_dicts(), + data=dff.to_dicts(), columns=[ { "name": columns[i], @@ -193,7 +198,7 @@ def get_sources_tables(source_path) -> html.Div: "type": "text", "format": {"nully": "N/A"}, } - for i in df.schema.names() + for i in dff.schema.names() ], style_cell_conditional=[ { @@ -416,6 +421,121 @@ def get_duplicate_matrix() -> html.Div: ) +def get_geographic_maps(dff: pl.DataFrame) -> list | None: + """ + Génère les cartes géographiques pour la métropole et les DOM-TOM. + """ + + # Seulement si les données ne sont pas trop importantes + + if dff.height > 10000: + return [] + + # Liste des codes départements Outre-Mer + dom_codes = ["971", "972", "973", "974", "976"] + + # Couleurs accessibles (Okabe-Ito) + color_acheteur = "#E69F00" # Orange + color_titulaire = "#56B4E9" # Bleu ciel + + regions = { + "Métropole": dff.filter(~pl.col("acheteur_departement_code").is_in(dom_codes)) + } + + # Ajout des DOM s'ils ont des données + for code in dom_codes: + dom_data = dff.filter(pl.col("acheteur_departement_code") == code) + if dom_data.height > 0: + name = f"Département {code}" + if code == "971": + name = "Guadeloupe" + elif code == "972": + name = "Martinique" + elif code == "973": + name = "Guyane" + elif code == "974": + name = "La Réunion" + elif code == "976": + name = "Mayotte" + regions[name] = dom_data + + cols = [] + for name, region_df in regions.items(): + fig = go.Figure() + + # Trace Acheteurs + mask_acheteur = region_df.filter( + pl.col("acheteur_latitude").is_not_null() + & pl.col("acheteur_longitude").is_not_null() + ) + if mask_acheteur.height > 0: + fig.add_trace( + go.Scattergeo( + lat=mask_acheteur["acheteur_latitude"], + lon=mask_acheteur["acheteur_longitude"], + mode="markers", + marker=dict(size=6, color=color_acheteur, opacity=0.5), + name="Acheteurs", + text=mask_acheteur["acheteur_nom"], + ) + ) + + # Trace Titulaires + mask_titulaire = region_df.filter( + pl.col("titulaire_latitude").is_not_null() + & pl.col("titulaire_longitude").is_not_null() + ) + if mask_titulaire.height > 0: + fig.add_trace( + go.Scattergeo( + lat=mask_titulaire["titulaire_latitude"], + lon=mask_titulaire["titulaire_longitude"], + mode="markers", + marker=dict(size=6, color=color_titulaire, opacity=0.5), + name="Titulaires", + text=mask_titulaire["titulaire_nom"], + ) + ) + + # Configuration spécifique de la vue + geo_config = dict( + projection_type="mercator", + showland=True, + landcolor="lightgray", + showcountries=True, + countrycolor="white", + fitbounds="locations" if name != "Métropole" else False, + resolution=50, + ) + + if name == "Métropole": + geo_config["lataxis_range"] = [41, 52] + geo_config["lonaxis_range"] = [-5, 10] + + fig.update_layout( + title=name, + geo=geo_config, + margin=dict(l=0, r=0, t=30, b=0), + height=400 if name == "Métropole" else 300, + showlegend=(name == "Métropole"), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), + ) + + # Taille de la colonne : 4 slots (md=12) pour Métropole, 1 slot (md=3) pour DOM + # On suppose une grille de 12 colonnes où 1 card = md=3 + col_width = 12 if name == "Métropole" else 3 + cols.append( + dbc.Col( + dcc.Graph(figure=fig, config={"displayModeBar": False}), + width=12, + md=col_width, + className="mb-4", + ) + ) + + return cols + + def make_column_picker(page: str): table_data = [] table_columns = [ diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 058f660..1876dfa 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -1,14 +1,20 @@ -from datetime import datetime +from datetime import datetime, timedelta -from dash import dcc, html, register_page +import dash_bootstrap_components as dbc +import polars as pl +import polars.selectors as cs +from dash import Input, Output, callback, dcc, html, register_page from src.figures import ( - get_barchart_sources, - get_duplicate_matrix, - get_map_count_marches, - get_yearly_statistics, + get_geographic_maps, +) +from src.utils import ( + departements, + df, + format_number, + get_enum_values_as_dict, + meta_content, ) -from src.utils import df, format_number, get_statistics, meta_content name = "Statistiques" @@ -21,13 +27,21 @@ register_page( image_url=meta_content["image_url"], order=3, ) +options_years = {} +for year in reversed(range(2017, datetime.now().year + 1)): + year = str(year) + options_years[year] = year + +options_departements = {} +for code, obj in departements.items(): + options_departements[code] = f"{obj['departement']} ({code})" -statistics: dict = get_statistics() -today_str = datetime.fromisoformat(statistics["datetime"]).strftime("%d/%m/%Y") layout = [ + dcc.Store(id="dashboard-filters"), + dcc.Location(id="dashboard_url"), html.Div( - className="container", + className="container-fluid", children=[ html.H2(name), dcc.Loading( @@ -35,51 +49,136 @@ layout = [ id="loading-statistques", type="default", children=[ - html.Div( - children=[ - dcc.Markdown(f""" - La publication de données essentielles de marchés publics (DECP) est souvent effectuée par - les plateformes de marchés publics (profils d'acheteurs). Cependant, certaines plateformes ne publient pas, - ou publient d'une manière qui rend la récupération des données compliquée. Les données présentées sur ce site - ne représentent donc pas tous les marchés attribués en France, seulement une partie significative. - - L'ajout de nouvelles plateformes [est en cours](https://github.com/ColinMaudry/decp-processing/issues?q=is%3Aissue%20label%3A%22source%20de%20donn%C3%A9es%22), - toutes les [contributions](/a-propos#contribuer) sont les bienvenues pour atteindre l'exhaustivité. - - Les statistiques publiées sur cette page ont été produites automatiquement à partir des données les plus récentes ({today_str}). - """), - html.H3( - "Statistiques générales sur les marchés", - id="marches", + dbc.Row( + [ + dbc.Col( + width=12, + md=3, + id="filters", + children=[ + html.H5("Période d'attribution"), + dbc.Row( + dcc.Dropdown( + id="dashboard_year", + options=options_years, + placeholder="12 derniers mois", + ), + ), + html.H5("Acheteur"), + dbc.Row( + dcc.Dropdown( + id="dashboard_acheteur_categorie", + options=get_enum_values_as_dict( + "acheteur_categorie" + ), + placeholder="Catégorie d'acheteur", + ), + ), + dbc.Row( + dcc.Dropdown( + id="dashboard_acheteur_departement_code", + searchable=True, + multi=True, + placeholder="Code département acheteur", + options=options_departements, + ), + ), + ], ), - html.P( - "À noter qu'une fois un marché attribué ses données essentielles peuvent malheureusement mettre plusieurs mois à être publiées par l'acheteur." + dbc.Col( + width=12, + md=9, + id="cards", + children=[ + dbc.Row( + ( + dbc.Col( + width=6, + md=4, + className="card", + id="card_basic_counts", + ) + ), + className="mb-4", + ), + dbc.Row(id="maps_row"), + ], ), - html.H4("Statistiques cumulées"), - dcc.Markdown(f""" - - Nombre de marchés publics et accord-cadres : {format_number(statistics["nb_marches"])} - - Nombre d'acheteurs publics (SIRET) : {format_number(statistics["nb_acheteurs_uniques"])} - - Nombre de titulaires (SIRET) : {format_number(statistics["nb_titulaires_uniques"])} - - Je ne publie pas encore de statistiques sur les montants de marchés car je n'ai pas encore trouvé la bonne formule pour traiter les trop nombreux montants fantaisistes qui polluent les calculs. - """), - html.H4("Statistiques par année"), - get_yearly_statistics(statistics, today_str), - dcc.Graph(figure=get_map_count_marches()), - get_duplicate_matrix(), - html.H3("Nombre de marchés par source dans le temps"), - dcc.Graph( - figure=get_barchart_sources(df, "dateNotification") - ), - dcc.Graph( - figure=get_barchart_sources( - df, "datePublicationDonnees" - ) - ), - ], + ] ) ], ), ], - ) + ), ] + + +@callback( + Output("card_basic_counts", "children"), + Output("maps_row", "children"), + Input("dashboard_year", "value"), + Input("dashboard_acheteur_categorie", "value"), + Input("dashboard_acheteur_departement_code", "value"), +) +def udpate_dashboard_cards( + dashboard_year, dashboard_acheteur_categorie, dashboard_acheteur_departement_code +): + lff: pl.LazyFrame = df.lazy() + lff = lff.select( + "uid", + cs.starts_with("acheteur"), + cs.starts_with("titulaire"), + "dateNotification", + "montant", + ) + + # Application des filtres + + if dashboard_year: + lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) + else: + lff = lff.filter( + pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) + ) + + if dashboard_acheteur_categorie: + lff = lff.filter( + pl.col("acheteur_categorie").is_in(dashboard_acheteur_categorie) + ) + + if ( + dashboard_acheteur_departement_code + and len(dashboard_acheteur_departement_code) >= 2 + ): + lff = lff.filter( + pl.col("acheteur_departement_code").is_in( + dashboard_acheteur_departement_code + ) + ) + + # Génération des métriques + dff = lff.collect() + + nb_acheteurs = dff.select("acheteur_id").n_unique() + nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() + + df_per_uid = ( + dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first()) + ) + total_montant = df_per_uid.select(pl.col("montant").sum()).item() + nb_marches = df_per_uid.height + + card_basic_counts = [ + html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), + html.P( + ["Nombre d'acheteurs : ", html.Strong(str(format_number(nb_acheteurs)))] + ), + html.P( + ["Nombre de titulaires : ", html.Strong(str(format_number(nb_titulaires)))] + ), + html.P(["Montant total : ", html.Strong(format_number(total_montant) + " €")]), + ] + + geographic_maps = get_geographic_maps(dff) + + return card_basic_counts, geographic_maps diff --git a/src/utils.py b/src/utils.py index 8c9dae7..992a99b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -685,6 +685,16 @@ def get_button_properties(height): return download_disabled, download_text, download_title +def get_enum_values_as_dict(column_name): + for column in data_schema: + if column == column_name: + options = {} + for value in data_schema[column]["enum"]: + options[value] = value + return options + return {"not_found": "not found"} + + def invert_columns(columns): """ Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa. From d9f97cf8b3c02e6b15e8b08766e49b86ab6a1a30 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 13 Mar 2026 23:59:54 +0100 Subject: [PATCH 02/75] =?UTF-8?q?Modification=20de=20map=5Fcount=5Fmarches?= =?UTF-8?q?=20(plus=20efficace,=20utilise=20le=20d=C3=A9p=20de=20l'acheteu?= =?UTF-8?q?r)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/figures.py b/src/figures.py index f06ae37..008c1cf 100644 --- a/src/figures.py +++ b/src/figures.py @@ -11,20 +11,20 @@ from dash import dash_table, dcc, html from src.utils import data_schema, df, format_number -def get_map_count_marches(): - lf = df.lazy() - lf = lf.with_columns( - pl.col("lieuExecution_code").str.head(2).str.zfill(2).alias("Département") - ) - lf = ( - lf.select(["uid", "Département"]) +def get_map_count_marches(dff: pl.DataFrame) -> go.Figure: + lff: pl.LazyFrame = dff.lazy() + lff = lff.rename({"acheteur_departement_code": "Département"}) + + lff = ( + lff.select(["uid", "Département"]) .drop_nulls() - .unique(subset="uid") + .group_by("uid") + .agg(pl.col("Département").first()) .group_by("Département") .len("uid") ) # Suppression des infos pour les DOM/TOM pour l'instant - lf = lf.remove(pl.col("Département").is_in(["97", "98"])) + lff = lff.filter(~pl.col("Département").str.head(2).is_in(["97", "98"])) with open("./data/departements-1000m.geojson") as f: departements = json.load(f) @@ -33,7 +33,7 @@ def get_map_count_marches(): for f in departements["features"]: f["id"] = f["properties"]["code"] - df_map = lf.collect(engine="streaming") + df_map = lff.collect(engine="streaming") fig = px.choropleth( df_map, From 4771d14744b137ceecbb5c7e3f88ecb23ab8f59c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 14 Mar 2026 00:00:15 +0100 Subject: [PATCH 03/75] =?UTF-8?q?Utilise=20map=5Fcount=5Fmarches=20si=20tr?= =?UTF-8?q?op=20de=20march=C3=A9s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/figures.py b/src/figures.py index 008c1cf..c95be13 100644 --- a/src/figures.py +++ b/src/figures.py @@ -426,10 +426,18 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: Génère les cartes géographiques pour la métropole et les DOM-TOM. """ - # Seulement si les données ne sont pas trop importantes - - if dff.height > 10000: - return [] + # Si les données sont trop importantes on utilise une carte chloropleth + if dff.height > 5000: + return [ + dbc.Col( + dcc.Graph( + figure=get_map_count_marches(dff), config={"displayModeBar": False} + ), + width=12, + md=12, + className="mb-4", + ) + ] # Liste des codes départements Outre-Mer dom_codes = ["971", "972", "973", "974", "976"] From f0f9d8cb3df543f4f582a910595398c86a25ce74 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 14 Mar 2026 00:15:53 +0100 Subject: [PATCH 04/75] dashboard_acheteur_departement_code est une liste --- src/pages/statistiques.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 1876dfa..0237abb 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -146,10 +146,7 @@ def udpate_dashboard_cards( pl.col("acheteur_categorie").is_in(dashboard_acheteur_categorie) ) - if ( - dashboard_acheteur_departement_code - and len(dashboard_acheteur_departement_code) >= 2 - ): + if dashboard_acheteur_departement_code: lff = lff.filter( pl.col("acheteur_departement_code").is_in( dashboard_acheteur_departement_code From 302d253e6e9211aa5d5c3e8716a0f51d21ce2150 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 14 Mar 2026 00:28:02 +0100 Subject: [PATCH 05/75] =?UTF-8?q?Utilisation=20de=20cluster=20de=20marqueu?= =?UTF-8?q?rs=20gr=C3=A2ce=20=C3=A0=20dash-leaflet=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 +- src/assets/dash_clientside.js | 23 ++++++ src/figures.py | 145 +++++++++++++++++++++------------- src/pages/statistiques.py | 4 +- 4 files changed, 116 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cdca66b..2befbcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,9 @@ dependencies = [ "plotly[express]", "httpx", "pandas", # utilisé pour la création de certains graphiques - "unidecode" + "unidecode", + "dash-leaflet", + "dash-extensions" ] [project.optional-dependencies] diff --git a/src/assets/dash_clientside.js b/src/assets/dash_clientside.js index efef901..babba1c 100644 --- a/src/assets/dash_clientside.js +++ b/src/assets/dash_clientside.js @@ -1,4 +1,27 @@ window.dash_clientside = Object.assign({}, window.dash_clientside, { + leaflet: { + pointToLayer: function (feature, latlng, context) { + return L.circleMarker(latlng, { + radius: 5, + fillColor: feature.properties.marker_color, + color: "white", + weight: 1, + opacity: 1, + fillOpacity: 0.8, + }).bindTooltip(feature.properties.tooltip); + }, + clusterToLayer: function (feature, latlng, index, context) { + const count = feature.properties.point_count; + const size = count < 100 ? 30 : count < 1000 ? 40 : 50; + const color = "#333"; // Default cluster color + const icon = L.divIcon({ + html: `
${count}
`, + className: "marker-cluster", + iconSize: L.point(size, size), + }); + return L.marker(latlng, { icon: icon }); + }, + }, clientside: { clean_filters: function (trigger) { if (!trigger) { diff --git a/src/figures.py b/src/figures.py index c95be13..15319b8 100644 --- a/src/figures.py +++ b/src/figures.py @@ -3,10 +3,13 @@ from typing import Literal from urllib.error import HTTPError, URLError import dash_bootstrap_components as dbc +import dash_leaflet as dl +import dash_leaflet.express as dlx import plotly.express as px import plotly.graph_objects as go import polars as pl from dash import dash_table, dcc, html +from dash_extensions.javascript import Namespace from src.utils import data_schema, df, format_number @@ -452,8 +455,8 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: # Ajout des DOM s'ils ont des données for code in dom_codes: - dom_data = dff.filter(pl.col("acheteur_departement_code") == code) - if dom_data.height > 0: + dff_dom = dff.filter(pl.col("acheteur_departement_code") == code) + if dff_dom.height > 0: name = f"Département {code}" if code == "971": name = "Guadeloupe" @@ -465,76 +468,106 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: name = "La Réunion" elif code == "976": name = "Mayotte" - regions[name] = dom_data + regions[name] = dff_dom + + # Region centers for dash-leaflet + region_centers = { + "Métropole": ([46.6, 2.2], 6), + "Guadeloupe": ([16.23, -61.55], 9), + "Martinique": ([14.64, -61.02], 10), + "Guyane": ([3.93, -53.12], 7), + "La Réunion": ([-21.11, 55.53], 10), + "Mayotte": ([-12.82, 45.16], 11), + } + + # JavaScript functions for styling + ns = Namespace("dash_clientside", "leaflet") + point_to_layer = ns("pointToLayer") + cluster_to_layer = ns("clusterToLayer") cols = [] for name, region_df in regions.items(): - fig = go.Figure() + # Prepare data for GeoJSON + marker_dicts = [] # Trace Acheteurs - mask_acheteur = region_df.filter( - pl.col("acheteur_latitude").is_not_null() - & pl.col("acheteur_longitude").is_not_null() - ) - if mask_acheteur.height > 0: - fig.add_trace( - go.Scattergeo( - lat=mask_acheteur["acheteur_latitude"], - lon=mask_acheteur["acheteur_longitude"], - mode="markers", - marker=dict(size=6, color=color_acheteur, opacity=0.5), - name="Acheteurs", - text=mask_acheteur["acheteur_nom"], - ) + mask_acheteur = ( + region_df.select( + "uid", "acheteur_longitude", "acheteur_latitude", "acheteur_nom" ) + .group_by("acheteur_longitude", "acheteur_latitude", "acheteur_nom") + .len("nb_marches") + .filter( + pl.col("acheteur_latitude").is_not_null() + & pl.col("acheteur_longitude").is_not_null() + ) + ) + + if mask_acheteur.height > 0: + for row in mask_acheteur.to_dicts(): + marker_dicts.append( + { + "lat": row["acheteur_latitude"], + "lon": row["acheteur_longitude"], + "tooltip": f"{row['acheteur_nom']} ({row['nb_marches']} marchés)", + "marker_color": color_acheteur, + } + ) # Trace Titulaires - mask_titulaire = region_df.filter( - pl.col("titulaire_latitude").is_not_null() - & pl.col("titulaire_longitude").is_not_null() - ) - if mask_titulaire.height > 0: - fig.add_trace( - go.Scattergeo( - lat=mask_titulaire["titulaire_latitude"], - lon=mask_titulaire["titulaire_longitude"], - mode="markers", - marker=dict(size=6, color=color_titulaire, opacity=0.5), - name="Titulaires", - text=mask_titulaire["titulaire_nom"], - ) + mask_titulaire = ( + region_df.select( + "uid", "titulaire_longitude", "titulaire_latitude", "titulaire_nom" + ) + .group_by("titulaire_longitude", "titulaire_latitude", "titulaire_nom") + .len("nb_marches") + .filter( + pl.col("titulaire_latitude").is_not_null() + & pl.col("titulaire_longitude").is_not_null() ) - - # Configuration spécifique de la vue - geo_config = dict( - projection_type="mercator", - showland=True, - landcolor="lightgray", - showcountries=True, - countrycolor="white", - fitbounds="locations" if name != "Métropole" else False, - resolution=50, ) - if name == "Métropole": - geo_config["lataxis_range"] = [41, 52] - geo_config["lonaxis_range"] = [-5, 10] + if mask_titulaire.height > 0: + for row in mask_titulaire.to_dicts(): + marker_dicts.append( + { + "lat": row["titulaire_latitude"], + "lon": row["titulaire_longitude"], + "tooltip": f"{row['titulaire_nom']} ({row['nb_marches']} marchés)", + "marker_color": color_titulaire, + } + ) - fig.update_layout( - title=name, - geo=geo_config, - margin=dict(l=0, r=0, t=30, b=0), - height=400 if name == "Métropole" else 300, - showlegend=(name == "Métropole"), - legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), - ) + geojson_data = dlx.dicts_to_geojson(marker_dicts) - # Taille de la colonne : 4 slots (md=12) pour Métropole, 1 slot (md=3) pour DOM - # On suppose une grille de 12 colonnes où 1 card = md=3 + center, zoom = region_centers.get(name, ([46.6, 2.2], 6)) col_width = 12 if name == "Métropole" else 3 + region_id = name.lower().replace(" ", "-") cols.append( dbc.Col( - dcc.Graph(figure=fig, config={"displayModeBar": False}), + [ + html.H5(name), + dl.Map( + [ + dl.TileLayer(), + dl.GeoJSON( + data=geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}", + ), + ], + center=center, + zoom=zoom, + style={ + "width": "100%", + "height": "400px" if name == "Métropole" else "300px", + }, + id=f"map-{region_id}", + ), + ], width=12, md=col_width, className="mb-4", diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 0237abb..1a74231 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -142,9 +142,7 @@ def udpate_dashboard_cards( ) if dashboard_acheteur_categorie: - lff = lff.filter( - pl.col("acheteur_categorie").is_in(dashboard_acheteur_categorie) - ) + lff = lff.filter(pl.col("acheteur_categorie") == dashboard_acheteur_categorie) if dashboard_acheteur_departement_code: lff = lff.filter( From adc8457abccfbe50f7cbea9873f282efda678962 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 15 Mar 2026 16:05:33 +0100 Subject: [PATCH 06/75] Cartes avec cluster de points si > lignes #65 --- src/assets/dash_clientside.js | 8 +++- src/figures.py | 70 ++++++++++++++++++++++++----------- src/pages/statistiques.py | 1 + 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/assets/dash_clientside.js b/src/assets/dash_clientside.js index babba1c..2c41e6a 100644 --- a/src/assets/dash_clientside.js +++ b/src/assets/dash_clientside.js @@ -11,11 +11,15 @@ window.dash_clientside = Object.assign({}, window.dash_clientside, { }).bindTooltip(feature.properties.tooltip); }, clusterToLayer: function (feature, latlng, index, context) { + console.log(feature); + console.log(index); + console.log(context); + const count = feature.properties.point_count; const size = count < 100 ? 30 : count < 1000 ? 40 : 50; - const color = "#333"; // Default cluster color + const color = "#555"; // Default cluster color const icon = L.divIcon({ - html: `
${count}
`, + html: `
${count}
`, className: "marker-cluster", iconSize: L.point(size, size), }); diff --git a/src/figures.py b/src/figures.py index 15319b8..a63cab8 100644 --- a/src/figures.py +++ b/src/figures.py @@ -430,7 +430,7 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: """ # Si les données sont trop importantes on utilise une carte chloropleth - if dff.height > 5000: + if dff.height > 50000: return [ dbc.Col( dcc.Graph( @@ -443,21 +443,30 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: ] # Liste des codes départements Outre-Mer - dom_codes = ["971", "972", "973", "974", "976"] + region_codes: list = ["Métropole", "971", "972", "973", "974", "976"] + dom_codes = region_codes[1:] # Couleurs accessibles (Okabe-Ito) color_acheteur = "#E69F00" # Orange color_titulaire = "#56B4E9" # Bleu ciel - regions = { - "Métropole": dff.filter(~pl.col("acheteur_departement_code").is_in(dom_codes)) - } + regions = {} # Ajout des DOM s'ils ont des données - for code in dom_codes: - dff_dom = dff.filter(pl.col("acheteur_departement_code") == code) - if dff_dom.height > 0: - name = f"Département {code}" + for code in region_codes: + if code == "Métropole": + dff_region = dff.filter( + ( + ~pl.col("acheteur_departement_code").is_in(dom_codes) + | (~pl.col("titulaire_departement_code").is_in(dom_codes)) + ) + ) + else: + dff_region = dff.filter( + (pl.col("acheteur_departement_code") == code) + | (pl.col("titulaire_departement_code") == code) + ) + if dff_region.height > 0: if code == "971": name = "Guadeloupe" elif code == "972": @@ -468,16 +477,21 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: name = "La Réunion" elif code == "976": name = "Mayotte" - regions[name] = dff_dom + elif code == "Métropole": + name = "Métropole" + else: + name = f"Département {code}" + + regions[name] = dff_region # Region centers for dash-leaflet region_centers = { - "Métropole": ([46.6, 2.2], 6), + "Métropole": ([46.6, 2.2], 5), "Guadeloupe": ([16.23, -61.55], 9), "Martinique": ([14.64, -61.02], 10), "Guyane": ([3.93, -53.12], 7), - "La Réunion": ([-21.11, 55.53], 10), - "Mayotte": ([-12.82, 45.16], 11), + "La Réunion": ([-21.11, 55.53], 9), + "Mayotte": ([-12.82, 45.16], 10), } # JavaScript functions for styling @@ -487,9 +501,6 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: cols = [] for name, region_df in regions.items(): - # Prepare data for GeoJSON - marker_dicts = [] - # Trace Acheteurs mask_acheteur = ( region_df.select( @@ -503,9 +514,11 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: ) ) + acheteurs_marker_dicts = [] + if mask_acheteur.height > 0: for row in mask_acheteur.to_dicts(): - marker_dicts.append( + acheteurs_marker_dicts.append( { "lat": row["acheteur_latitude"], "lon": row["acheteur_longitude"], @@ -527,9 +540,11 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: ) ) + titulaires_marker_dicts = [] + if mask_titulaire.height > 0: for row in mask_titulaire.to_dicts(): - marker_dicts.append( + titulaires_marker_dicts.append( { "lat": row["titulaire_latitude"], "lon": row["titulaire_longitude"], @@ -538,10 +553,11 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: } ) - geojson_data = dlx.dicts_to_geojson(marker_dicts) + acheteurs_geojson_data = dlx.dicts_to_geojson(acheteurs_marker_dicts) + titulaires_geojson_data = dlx.dicts_to_geojson(titulaires_marker_dicts) center, zoom = region_centers.get(name, ([46.6, 2.2], 6)) - col_width = 12 if name == "Métropole" else 3 + col_width = 6 if name == "Métropole" else 3 region_id = name.lower().replace(" ", "-") cols.append( dbc.Col( @@ -551,12 +567,22 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: [ dl.TileLayer(), dl.GeoJSON( - data=geojson_data, + data=titulaires_geojson_data, cluster=True, zoomToBoundsOnClick=True, pointToLayer=point_to_layer, clusterToLayer=cluster_to_layer, - id=f"geojson-{region_id}", + id=f"geojson-{region_id}-titulaires", + options={"fillColor": color_titulaire}, + ), + dl.GeoJSON( + data=acheteurs_geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}-acheteurs", + options={"fillColor": color_acheteur}, ), ], center=center, diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 1a74231..99fcbc7 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -163,6 +163,7 @@ def udpate_dashboard_cards( total_montant = df_per_uid.select(pl.col("montant").sum()).item() nb_marches = df_per_uid.height + # À transformer en fonction card_basic_counts = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), html.P( From f531ce70918fa543b38750e4f0c9f61f2e14360a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 16 Mar 2026 18:06:32 +0100 Subject: [PATCH 07/75] =?UTF-8?q?Choix=20de=20carte=20dynamique,=20m=C3=AA?= =?UTF-8?q?me=20pour=20les=20TOM=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 416 +++++++++++++++++++++++++------------------------ src/utils.py | 12 ++ 2 files changed, 226 insertions(+), 202 deletions(-) diff --git a/src/figures.py b/src/figures.py index a63cab8..1d3a132 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,4 +1,3 @@ -import json from typing import Literal from urllib.error import HTTPError, URLError @@ -11,57 +10,7 @@ import polars as pl from dash import dash_table, dcc, html from dash_extensions.javascript import Namespace -from src.utils import data_schema, df, format_number - - -def get_map_count_marches(dff: pl.DataFrame) -> go.Figure: - lff: pl.LazyFrame = dff.lazy() - lff = lff.rename({"acheteur_departement_code": "Département"}) - - lff = ( - lff.select(["uid", "Département"]) - .drop_nulls() - .group_by("uid") - .agg(pl.col("Département").first()) - .group_by("Département") - .len("uid") - ) - # Suppression des infos pour les DOM/TOM pour l'instant - lff = lff.filter(~pl.col("Département").str.head(2).is_in(["97", "98"])) - - with open("./data/departements-1000m.geojson") as f: - departements = json.load(f) - - # Ajout de feature.id - for f in departements["features"]: - f["id"] = f["properties"]["code"] - - df_map = lff.collect(engine="streaming") - - fig = px.choropleth( - df_map, - geojson=departements, - locations="Département", - color="uid", - color_continuous_scale="Reds", - title="Nombres de marchés attribués par département (lieu d'exécution)", - range_color=(df_map["uid"].min(), df_map["uid"].max()), - labels={"uid": "Marchés attribués"}, - scope="europe", - width=900, - height=700, - ) - - fig.update_geos(fitbounds="locations", visible=False) - fig.update_layout( - mapbox={ - "style": "carto-positron", - "center": {"lon": 10, "lat": 10}, - "zoom": 1, - "domain": {"x": [0, 1], "y": [0, 1]}, - } - ) - return fig +from src.utils import data_schema, departements_geojson, df, format_number def get_yearly_statistics(statistics, today_str) -> html.Div: @@ -82,11 +31,11 @@ def get_yearly_statistics(statistics, today_str) -> html.Div: } ) - df = pl.DataFrame(data) + dff = pl.DataFrame(data) # Create Dash DataTable table = dash_table.DataTable( - data=df.to_dicts(), + data=dff.to_dicts(), columns=[ {"name": "Année", "id": "Année"}, {"name": "Marchés et accord-cadres", "id": "Marchés et accord-cadres"}, @@ -429,178 +378,241 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: Génère les cartes géographiques pour la métropole et les DOM-TOM. """ - # Si les données sont trop importantes on utilise une carte chloropleth - if dff.height > 50000: - return [ - dbc.Col( - dcc.Graph( - figure=get_map_count_marches(dff), config={"displayModeBar": False} - ), - width=12, - md=12, - className="mb-4", - ) - ] + regions: dict = { + "Métropole": { + "coordinates": [46.6, 2.2], + "zoom_leaflet": 5, + "zoom_chloropleth": 1, + "name": "Métropole", + }, + "971": { + "coordinates": [16.23, -61.55], + "zoom_leaflet": 9, + "zoom_chloropleth": 1, + "name": "Guadeloupe", + }, + "972": { + "coordinates": [14.64, -61.02], + "zoom_leaflet": 10, + "zoom_chloropleth": 1, + "name": "Martinique", + }, + "973": { + "coordinates": [3.93, -53.12], + "zoom_leaflet": 7, + "zoom_chloropleth": 1, + "name": "Guyane", + }, + "974": { + "coordinates": [-21.11, 55.53], + "zoom_leaflet": 9, + "zoom_chloropleth": 1, + "name": "La Réunion", + }, + "976": { + "coordinates": [-12.82, 45.16], + "zoom_leaflet": 10, + "zoom_chloropleth": 1, + "name": "Mayotte", + }, + } # Liste des codes départements Outre-Mer - region_codes: list = ["Métropole", "971", "972", "973", "974", "976"] - dom_codes = region_codes[1:] + dom_codes = [code for code in regions.keys() if code != "Métropole"] + print("dom codes", dom_codes) - # Couleurs accessibles (Okabe-Ito) - color_acheteur = "#E69F00" # Orange - color_titulaire = "#56B4E9" # Bleu ciel - - regions = {} - - # Ajout des DOM s'ils ont des données - for code in region_codes: - if code == "Métropole": - dff_region = dff.filter( + def make_map_data(region_code: str) -> tuple[list, str or None]: + lff: pl.LazyFrame = dff.lazy() + if region_code == "Métropole": + lff = lff.filter( ( ~pl.col("acheteur_departement_code").is_in(dom_codes) | (~pl.col("titulaire_departement_code").is_in(dom_codes)) ) ) else: - dff_region = dff.filter( + lff = lff.filter( (pl.col("acheteur_departement_code") == code) | (pl.col("titulaire_departement_code") == code) ) - if dff_region.height > 0: - if code == "971": - name = "Guadeloupe" - elif code == "972": - name = "Martinique" - elif code == "973": - name = "Guyane" - elif code == "974": - name = "La Réunion" - elif code == "976": - name = "Mayotte" - elif code == "Métropole": - name = "Métropole" - else: - name = f"Département {code}" - regions[name] = dff_region + nb_marches = lff.select("uid").group_by("uid").first().collect().height - # Region centers for dash-leaflet - region_centers = { - "Métropole": ([46.6, 2.2], 5), - "Guadeloupe": ([16.23, -61.55], 9), - "Martinique": ([14.64, -61.02], 10), - "Guyane": ([3.93, -53.12], 7), - "La Réunion": ([-21.11, 55.53], 9), - "Mayotte": ([-12.82, 45.16], 10), - } + if nb_marches == 0: + return [], None + dfs = [] + + if (code == "Métropole" and nb_marches > 20000) or ( + code != "Métropole" and nb_marches > 10000 + ): + _map_type: str = "chloropleth" + + lff = lff.rename({"acheteur_departement_code": "Département"}) + lff = ( + lff.select(["uid", "Département"]) + .drop_nulls() + .group_by("uid") + .agg(pl.col("Département").first()) + .group_by("Département") + .len("uid") + ) + dfs.append(lff.collect()) + else: + _map_type: str = "clusters" + for org_type in ["acheteur", "titulaire"]: + lff_org = ( + lff.select( + "uid", + f"{org_type}_longitude", + f"{org_type}_latitude", + f"{org_type}_nom", + ) + .group_by( + f"{org_type}_longitude", + f"{org_type}_latitude", + f"{org_type}_nom", + ) + .len("nb_marches") + .filter( + pl.col(f"{org_type}_latitude").is_not_null() + & pl.col(f"{org_type}_longitude").is_not_null() + ) + ) + + markers = [] + + # Couleurs accessibles (Okabe-Ito) + colors = { + "acheteur": "#E69F00", # orange + "titulaire": "#56B4E9", # bleu ciel + } + + for row in lff_org.collect().to_dicts(): + markers.append( + { + "lat": row[f"{org_type}_latitude"], + "lon": row[f"{org_type}_longitude"], + "tooltip": f"{row[f'{org_type}_nom']} ({row['nb_marches']} marchés)", + "marker_color": colors[org_type], + } + ) + dfs.append(markers) + + return dfs, _map_type + + cols = [] + + for code in regions.keys(): + regions[code]["data"], map_type = make_map_data(code) + print("region", regions[code]["name"], len(regions[code]["data"][0]), map_type) + + if map_type == "chloropleth": + map_graph = make_chloropleth_map(regions[code]) # call chloropleth function + elif map_type == "clusters": + map_graph = make_clusters_map(regions[code]) + elif map_type is None: + continue + else: + raise ValueError(f"Map type '{map_type}' not recognised") + + col = dbc.Col( + children=[ + html.H5(regions[code]["name"]), + map_graph, + ], + md=6 if code == "Métropole" else 3, + width=12, + className="mb-4", + ) + cols.append(col) + + return cols + + +def make_chloropleth_map(region: dict) -> dcc.Graph: + df_map = region["data"][0] + + fig = px.choropleth( + df_map, + geojson=departements_geojson, + locations="Département", + color="uid", + color_continuous_scale="Reds", + title="Nombres de marchés attribués", + range_color=(df_map["uid"].min(), df_map["uid"].max()), + labels={"uid": "Marchés attribués"}, + scope="europe", + width=400, + height=400, + ) + + fig.update_geos(fitbounds="locations", visible=False) + fig.update_layout( + mapbox={ + "style": "carto-positron", + "center": {"lon": 10, "lat": 10}, + "zoom": 8, + "domain": {"x": [0, 1], "y": [0, 1]}, + } + ) + + graph = dcc.Graph(figure=fig, config={"displayModeBar": False}) + return graph + + +def make_clusters_map(region: dict): # JavaScript functions for styling ns = Namespace("dash_clientside", "leaflet") point_to_layer = ns("pointToLayer") cluster_to_layer = ns("clusterToLayer") - cols = [] - for name, region_df in regions.items(): - # Trace Acheteurs - mask_acheteur = ( - region_df.select( - "uid", "acheteur_longitude", "acheteur_latitude", "acheteur_nom" - ) - .group_by("acheteur_longitude", "acheteur_latitude", "acheteur_nom") - .len("nb_marches") - .filter( - pl.col("acheteur_latitude").is_not_null() - & pl.col("acheteur_longitude").is_not_null() - ) - ) + name = region["name"] - acheteurs_marker_dicts = [] + # Données de la région + region_acheteurs = region["data"][0] + region_titulaires = region["data"][1] - if mask_acheteur.height > 0: - for row in mask_acheteur.to_dicts(): - acheteurs_marker_dicts.append( - { - "lat": row["acheteur_latitude"], - "lon": row["acheteur_longitude"], - "tooltip": f"{row['acheteur_nom']} ({row['nb_marches']} marchés)", - "marker_color": color_acheteur, - } - ) + # Couleurs + color_acheteur = region_acheteurs[0]["marker_color"] + color_titulaire = region_titulaires[0]["marker_color"] - # Trace Titulaires - mask_titulaire = ( - region_df.select( - "uid", "titulaire_longitude", "titulaire_latitude", "titulaire_nom" - ) - .group_by("titulaire_longitude", "titulaire_latitude", "titulaire_nom") - .len("nb_marches") - .filter( - pl.col("titulaire_latitude").is_not_null() - & pl.col("titulaire_longitude").is_not_null() - ) - ) + acheteurs_geojson_data = dlx.dicts_to_geojson(region_acheteurs) + titulaires_geojson_data = dlx.dicts_to_geojson(region_titulaires) - titulaires_marker_dicts = [] - - if mask_titulaire.height > 0: - for row in mask_titulaire.to_dicts(): - titulaires_marker_dicts.append( - { - "lat": row["titulaire_latitude"], - "lon": row["titulaire_longitude"], - "tooltip": f"{row['titulaire_nom']} ({row['nb_marches']} marchés)", - "marker_color": color_titulaire, - } - ) - - acheteurs_geojson_data = dlx.dicts_to_geojson(acheteurs_marker_dicts) - titulaires_geojson_data = dlx.dicts_to_geojson(titulaires_marker_dicts) - - center, zoom = region_centers.get(name, ([46.6, 2.2], 6)) - col_width = 6 if name == "Métropole" else 3 - region_id = name.lower().replace(" ", "-") - cols.append( - dbc.Col( - [ - html.H5(name), - dl.Map( - [ - dl.TileLayer(), - dl.GeoJSON( - data=titulaires_geojson_data, - cluster=True, - zoomToBoundsOnClick=True, - pointToLayer=point_to_layer, - clusterToLayer=cluster_to_layer, - id=f"geojson-{region_id}-titulaires", - options={"fillColor": color_titulaire}, - ), - dl.GeoJSON( - data=acheteurs_geojson_data, - cluster=True, - zoomToBoundsOnClick=True, - pointToLayer=point_to_layer, - clusterToLayer=cluster_to_layer, - id=f"geojson-{region_id}-acheteurs", - options={"fillColor": color_acheteur}, - ), - ], - center=center, - zoom=zoom, - style={ - "width": "100%", - "height": "400px" if name == "Métropole" else "300px", - }, - id=f"map-{region_id}", - ), - ], - width=12, - md=col_width, - className="mb-4", - ) - ) - - return cols + center, zoom = region["coordinates"], region["zoom_leaflet"] + region_id = name.lower().replace(" ", "-") + leaflet_map = dl.Map( + [ + dl.TileLayer(), + dl.GeoJSON( + data=titulaires_geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}-titulaires", + options={"fillColor": color_titulaire}, + ), + dl.GeoJSON( + data=acheteurs_geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}-acheteurs", + options={"fillColor": color_acheteur}, + ), + ], + center=center, + zoom=zoom, + style={ + "width": "100%", + "height": "400px" if name == "Métropole" else "300px", + }, + id=f"map-{region_id}", + ) + return leaflet_map def make_column_picker(page: str): diff --git a/src/utils.py b/src/utils.py index 992a99b..b334d01 100644 --- a/src/utils.py +++ b/src/utils.py @@ -286,6 +286,17 @@ def get_departements() -> dict: return data +def get_departements_geojson() -> dict: + with open("./data/departements-1000m.geojson") as f: + geojson = json.load(f) + + # Ajout de feature.id + for f in geojson["features"]: + f["id"] = f["properties"]["code"] + + return geojson + + def get_departement_region(code_postal): if code_postal > "97000": code_departement = code_postal[:3] @@ -774,6 +785,7 @@ df_titulaires_marches: pl.DataFrame = ( ) departements = get_departements() +departements_geojson = get_departements_geojson() domain_name = ( "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" ) From ab3377ef60ef39804188a4ff56b96c86d2a4ba42 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 17:16:24 +0100 Subject: [PATCH 08/75] Grid de cards avec points de rupture #65 --- data/departements.json | 8 ++++++ src/figures.py | 57 +++++++++++++++++++++++---------------- src/pages/statistiques.py | 37 +++++++++---------------- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/data/departements.json b/data/departements.json index 4115895..1f346d0 100644 --- a/data/departements.json +++ b/data/departements.json @@ -391,8 +391,16 @@ "departement": "La Réunion", "region": "La Réunion" }, + "975": { + "departement": "Saint-Pierre-et-Miquelon", + "region": "Saint-Pierre-et-Miquelon" + }, "976": { "departement": "Mayotte", "region": "Mayotte" + }, + "977": { + "departement": "Saint-Barthelemy", + "region": "Saint-Barthelemy" } } diff --git a/src/figures.py b/src/figures.py index 1d3a132..556e802 100644 --- a/src/figures.py +++ b/src/figures.py @@ -417,18 +417,12 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: }, } - # Liste des codes départements Outre-Mer - dom_codes = [code for code in regions.keys() if code != "Métropole"] - print("dom codes", dom_codes) - def make_map_data(region_code: str) -> tuple[list, str or None]: lff: pl.LazyFrame = dff.lazy() if region_code == "Métropole": lff = lff.filter( - ( - ~pl.col("acheteur_departement_code").is_in(dom_codes) - | (~pl.col("titulaire_departement_code").is_in(dom_codes)) - ) + (pl.col("acheteur_departement_code").str.len_chars() == 2) + & (pl.col("titulaire_departement_code").str.len_chars() == 2) ) else: lff = lff.filter( @@ -436,14 +430,14 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: | (pl.col("titulaire_departement_code") == code) ) - nb_marches = lff.select("uid").group_by("uid").first().collect().height + nb_marches = lff.select("uid").collect()["uid"].n_unique() if nb_marches == 0: return [], None dfs = [] - if (code == "Métropole" and nb_marches > 20000) or ( + if (code == "Métropole" and nb_marches > 30000) or ( code != "Métropole" and nb_marches > 10000 ): _map_type: str = "chloropleth" @@ -505,10 +499,9 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: for code in regions.keys(): regions[code]["data"], map_type = make_map_data(code) - print("region", regions[code]["name"], len(regions[code]["data"][0]), map_type) if map_type == "chloropleth": - map_graph = make_chloropleth_map(regions[code]) # call chloropleth function + map_graph = make_chloropleth_map(regions[code]) elif map_type == "clusters": map_graph = make_clusters_map(regions[code]) elif map_type is None: @@ -516,15 +509,9 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: else: raise ValueError(f"Map type '{map_type}' not recognised") - col = dbc.Col( - children=[ - html.H5(regions[code]["name"]), - map_graph, - ], - md=6 if code == "Métropole" else 3, - width=12, - className="mb-4", - ) + md = 6 if code == "Métropole" else 3 + + col = make_card(regions[code]["name"], md=md, fig=map_graph) cols.append(col) return cols @@ -543,7 +530,6 @@ def make_chloropleth_map(region: dict) -> dcc.Graph: range_color=(df_map["uid"].min(), df_map["uid"].max()), labels={"uid": "Marchés attribués"}, scope="europe", - width=400, height=400, ) @@ -561,7 +547,7 @@ def make_chloropleth_map(region: dict) -> dcc.Graph: return graph -def make_clusters_map(region: dict): +def make_clusters_map(region: dict) -> dl.Map: # JavaScript functions for styling ns = Namespace("dash_clientside", "leaflet") point_to_layer = ns("pointToLayer") @@ -615,6 +601,31 @@ def make_clusters_map(region: dict): return leaflet_map +def make_card( + title: str, subtitle=None, fig=None, paragraphs=None, md=3, width=12 +) -> dbc.Col: + children = [] + if title: + children.append(html.H5(title, className="card-title")) + if subtitle: + children.append(html.H6(subtitle, className="card-subtitle mb-2 text-muted")) + if fig: + children.append(fig) + if paragraphs: + for p in paragraphs: + p.className = "card-text" + children.append(p) + + card = dbc.Col( + html.Div(html.Div(className="card-body", children=children), className="card"), + lg=6, + xl=4, + # width=width, + # className="mb-4", + ) + return card + + def make_column_picker(page: str): table_data = [] table_columns = [ diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 99fcbc7..df71edd 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -5,9 +5,7 @@ import polars as pl import polars.selectors as cs from dash import Input, Output, callback, dcc, html, register_page -from src.figures import ( - get_geographic_maps, -) +from src.figures import get_geographic_maps, make_card from src.utils import ( departements, df, @@ -52,8 +50,8 @@ layout = [ dbc.Row( [ dbc.Col( - width=12, - md=3, + xl=3, + lg=4, id="filters", children=[ html.H5("Période d'attribution"), @@ -87,22 +85,10 @@ layout = [ ), dbc.Col( width=12, - md=9, + lg=8, + xl=9, id="cards", - children=[ - dbc.Row( - ( - dbc.Col( - width=6, - md=4, - className="card", - id="card_basic_counts", - ) - ), - className="mb-4", - ), - dbc.Row(id="maps_row"), - ], + children=[], ), ] ) @@ -114,8 +100,7 @@ layout = [ @callback( - Output("card_basic_counts", "children"), - Output("maps_row", "children"), + Output("cards", "children"), Input("dashboard_year", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), @@ -163,6 +148,8 @@ def udpate_dashboard_cards( total_montant = df_per_uid.select(pl.col("montant").sum()).item() nb_marches = df_per_uid.height + cards = [] + # À transformer en fonction card_basic_counts = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), @@ -175,6 +162,8 @@ def udpate_dashboard_cards( html.P(["Montant total : ", html.Strong(format_number(total_montant) + " €")]), ] - geographic_maps = get_geographic_maps(dff) + cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) - return card_basic_counts, geographic_maps + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) + + return dbc.Row(children=cards + geographic_maps) From 9629231e299a4876370080450b0685d917c9f0d4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 17:30:41 +0100 Subject: [PATCH 09/75] Fixed masquage silencieux de la chloropleth #65 --- src/figures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/figures.py b/src/figures.py index 556e802..0f7f6b0 100644 --- a/src/figures.py +++ b/src/figures.py @@ -609,7 +609,7 @@ def make_card( children.append(html.H5(title, className="card-title")) if subtitle: children.append(html.H6(subtitle, className="card-subtitle mb-2 text-muted")) - if fig: + if fig is not None: children.append(fig) if paragraphs: for p in paragraphs: From e754a3a21784cf470b52c2e33fd9786dadbf4fdd Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 20:33:36 +0100 Subject: [PATCH 10/75] Configuration fine des tailles de cartes #65 --- src/assets/css/style.css | 5 +++++ src/figures.py | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index a183642..326523f 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -423,6 +423,11 @@ input[type="checkbox"] { } /* --- Organization Cards (Grid Items) --- */ + +#cards .card { + margin-bottom: 16px; +} + .org_title { grid-column: 1 / 3; grid-row: 1; diff --git a/src/figures.py b/src/figures.py index 0f7f6b0..215847f 100644 --- a/src/figures.py +++ b/src/figures.py @@ -509,9 +509,9 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: else: raise ValueError(f"Map type '{map_type}' not recognised") - md = 6 if code == "Métropole" else 3 + lg, xl = (12, 8) if code == "Métropole" else (6, 4) - col = make_card(regions[code]["name"], md=md, fig=map_graph) + col = make_card(regions[code]["name"], fig=map_graph, lg=lg, xl=xl) cols.append(col) return cols @@ -526,11 +526,9 @@ def make_chloropleth_map(region: dict) -> dcc.Graph: locations="Département", color="uid", color_continuous_scale="Reds", - title="Nombres de marchés attribués", range_color=(df_map["uid"].min(), df_map["uid"].max()), labels={"uid": "Marchés attribués"}, scope="europe", - height=400, ) fig.update_geos(fitbounds="locations", visible=False) @@ -602,7 +600,7 @@ def make_clusters_map(region: dict) -> dl.Map: def make_card( - title: str, subtitle=None, fig=None, paragraphs=None, md=3, width=12 + title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4 ) -> dbc.Col: children = [] if title: @@ -618,8 +616,8 @@ def make_card( card = dbc.Col( html.Div(html.Div(className="card-body", children=children), className="card"), - lg=6, - xl=4, + lg=lg, + xl=xl, # width=width, # className="mb-4", ) From 6c778882f94774f5bcf558810f4a13eb30dd2e92 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 20:51:09 +0100 Subject: [PATCH 11/75] =?UTF-8?q?Filtre=20par=20type=20de=20march=C3=A9=20?= =?UTF-8?q?#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/statistiques.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index df71edd..4a1f970 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -81,6 +81,14 @@ layout = [ options=options_departements, ), ), + html.H5("Marché"), + dbc.Row( + dcc.Dropdown( + id="dashboard_marche_type", + placeholder="Type de marché", + options=get_enum_values_as_dict("type"), + ), + ), ], ), dbc.Col( @@ -104,9 +112,13 @@ layout = [ Input("dashboard_year", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_marche_type", "value"), ) def udpate_dashboard_cards( - dashboard_year, dashboard_acheteur_categorie, dashboard_acheteur_departement_code + dashboard_year, + dashboard_acheteur_categorie, + dashboard_acheteur_departement_code, + dashboard_marche_type, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -136,6 +148,9 @@ def udpate_dashboard_cards( ) ) + if dashboard_marche_type: + lff = lff.filter(pl.col("type") == dashboard_marche_type) + # Génération des métriques dff = lff.collect() From c02fb995c54eeb05e39d83abd1f070a719bc3eb4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 22:44:15 +0100 Subject: [PATCH 12/75] =?UTF-8?q?Ajout=20de=20filtres=20et=20des=20donuts?= =?UTF-8?q?=20de=20cat=C3=A9gorie=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 7 ++++ src/figures.py | 19 +++++++++ src/pages/statistiques.py | 87 ++++++++++++++++++++++++++++++++++++--- src/utils.py | 14 +++---- 4 files changed, 114 insertions(+), 13 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 326523f..9705af4 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -190,6 +190,13 @@ p.version > a { grid-row: 1; } +/* --- Dashboard inputs --- */ + +.Select--multi .Select-value { + color: var(--primary-color); + background-color: rgba(255, 240, 240, 0.4); +} + /* --- Tables (Dash & Custom) --- */ /* Table Menu (Exports etc) */ diff --git a/src/figures.py b/src/figures.py index 215847f..c32f865 100644 --- a/src/figures.py +++ b/src/figures.py @@ -624,6 +624,25 @@ def make_card( return card +def make_donut(lff: pl.LazyFrame, names_col): + title = data_schema[names_col]["title"] + lff = lff.rename({names_col: title}) + lff = lff.select("uid", title) + lff = lff.group_by(title).len("Nombre") + lff = lff.with_columns(pl.col(title).replace(None, pl.lit("?"))) + fig = px.pie( + lff.collect(engine="streaming"), + values="Nombre", + names=title, + hole=0.4, + color_discrete_sequence=px.colors.qualitative.Safe, + ) + fig = fig.update_traces(texttemplate="%{label}
%{percent}") + fig = fig.update_layout(showlegend=False, font=dict(size=14)) + graph = dcc.Graph(figure=fig) + return graph + + def make_column_picker(page: str): table_data = [] table_columns = [ diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 4a1f970..9de6a29 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -5,7 +5,7 @@ import polars as pl import polars.selectors as cs from dash import Input, Output, callback, dcc, html, register_page -from src.figures import get_geographic_maps, make_card +from src.figures import get_geographic_maps, make_card, make_donut from src.utils import ( departements, df, @@ -69,7 +69,7 @@ layout = [ options=get_enum_values_as_dict( "acheteur_categorie" ), - placeholder="Catégorie d'acheteur", + placeholder="Catégorie", ), ), dbc.Row( @@ -81,14 +81,44 @@ layout = [ options=options_departements, ), ), + html.H5("Titulaire"), + dbc.Row( + dcc.Dropdown( + id="dashboard_titulaire_categorie", + placeholder="Catégorie", + options=get_enum_values_as_dict( + "titulaire_categorie" + ), + ), + ), html.H5("Marché"), dbc.Row( dcc.Dropdown( id="dashboard_marche_type", - placeholder="Type de marché", + placeholder="Type", options=get_enum_values_as_dict("type"), ), ), + dbc.Row( + dcc.Dropdown( + id="dashboard_marche_considerationsSociales", + placeholder="Considérations sociales", + options=get_enum_values_as_dict( + "considerationsSociales" + ), + multi=True, + ), + ), + dbc.Row( + dcc.Dropdown( + id="dashboard_marche_considerationsEnvironnementales", + placeholder="Considérations environnementales", + multi=True, + options=get_enum_values_as_dict( + "considerationsEnvironnementales" + ), + ), + ), ], ), dbc.Col( @@ -112,13 +142,19 @@ layout = [ Input("dashboard_year", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_titulaire_categorie", "value"), Input("dashboard_marche_type", "value"), + Input("dashboard_marche_considerationsSociales", "value"), + Input("dashboard_marche_considerationsEnvironnementales", "value"), ) def udpate_dashboard_cards( dashboard_year, dashboard_acheteur_categorie, dashboard_acheteur_departement_code, + dashboard_titulaire_categorie, dashboard_marche_type, + dashboard_marche_considerationsSociales, + dashboard_marche_considerationsEnvironnementales, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -127,10 +163,14 @@ def udpate_dashboard_cards( cs.starts_with("titulaire"), "dateNotification", "montant", + "considerationsSociales", + "considerationsEnvironnementales", ) # Application des filtres + ## Période + if dashboard_year: lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) else: @@ -138,6 +178,8 @@ def udpate_dashboard_cards( pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) ) + ## Acheteur + if dashboard_acheteur_categorie: lff = lff.filter(pl.col("acheteur_categorie") == dashboard_acheteur_categorie) @@ -148,12 +190,38 @@ def udpate_dashboard_cards( ) ) + ## Titulaire + + if dashboard_titulaire_categorie: + lff = lff.filter(pl.col("titulaire_categorie") == dashboard_titulaire_categorie) + + ## Marché + if dashboard_marche_type: lff = lff.filter(pl.col("type") == dashboard_marche_type) - # Génération des métriques - dff = lff.collect() + if dashboard_marche_considerationsSociales: + lff = lff.filter( + pl.col("considerationsSociales") + .str.split(", ") + .list.set_intersection(dashboard_marche_considerationsSociales) + .list.len() + > 0 + ) + if dashboard_marche_considerationsEnvironnementales: + lff = lff.filter( + pl.col("considerationsEnvironnementales") + .str.split(", ") + .list.set_intersection(dashboard_marche_considerationsEnvironnementales) + .list.len() + > 0 + ) + + # Génération des métriques + dff = lff.collect(engine="streaming") + + # À transformer en fonction nb_acheteurs = dff.select("acheteur_id").n_unique() nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() @@ -165,7 +233,6 @@ def udpate_dashboard_cards( cards = [] - # À transformer en fonction card_basic_counts = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), html.P( @@ -179,6 +246,14 @@ def udpate_dashboard_cards( cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) + donut_acheteur_categorie = make_donut(lff, "acheteur_categorie") + cards.append(make_card(title="Catégorie d'acheteur", fig=donut_acheteur_categorie)) + + donut_titulaire_categorie = make_donut(lff, "titulaire_categorie") + cards.append( + make_card(title="Catégorie d'entreprise", fig=donut_titulaire_categorie) + ) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) return dbc.Row(children=cards + geographic_maps) diff --git a/src/utils.py b/src/utils.py index b334d01..642bad4 100644 --- a/src/utils.py +++ b/src/utils.py @@ -697,13 +697,13 @@ def get_button_properties(height): def get_enum_values_as_dict(column_name): - for column in data_schema: - if column == column_name: - options = {} - for value in data_schema[column]["enum"]: - options[value] = value - return options - return {"not_found": "not found"} + try: + options = {} + for value in data_schema[column_name]["enum"]: + options[value] = value + return options + except KeyError: + return {"not_found": "not found"} def invert_columns(columns): From d50ec5b01e274e80d7f2219515175515dfc920b7 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 22:59:01 +0100 Subject: [PATCH 13/75] Statistiques => Observatoire #65 --- src/app.py | 4 ++-- src/pages/{statistiques.py => observatoire.py} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename src/pages/{statistiques.py => observatoire.py} (99%) diff --git a/src/app.py b/src/app.py index eb1931c..7a56058 100644 --- a/src/app.py +++ b/src/app.py @@ -53,7 +53,7 @@ def sitemap(): base_url = "https://decp.info" pages = [ "/", - "/statistiques", + "/observatoire", "/tableau", "/a-propos", ] @@ -160,7 +160,7 @@ navbar = dbc.Navbar( ) for page in page_registry.values() if page["name"] - in ["Recherche", "À propos", "Tableau", "Statistiques"] + in ["Recherche", "À propos", "Tableau", "Observatoire"] ], className="ms-auto", navbar=True, diff --git a/src/pages/statistiques.py b/src/pages/observatoire.py similarity index 99% rename from src/pages/statistiques.py rename to src/pages/observatoire.py index 9de6a29..eb726f7 100644 --- a/src/pages/statistiques.py +++ b/src/pages/observatoire.py @@ -14,12 +14,12 @@ from src.utils import ( meta_content, ) -name = "Statistiques" +name = "Observatoire" register_page( __name__, - path="/statistiques", - title="Statistiques | decp.info", + path="/observatoire", + title="Observatoire | decp.info", name=name, description="Visualisez l'état de la publication des données essentielles des marchés publics en France.", image_url=meta_content["image_url"], From 2f90a754ab32d4da701e76a55ef4e773ce82e855 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 13:38:38 +0100 Subject: [PATCH 14/75] =?UTF-8?q?Style=20inputs,=20ajout=20donut=20type=20?= =?UTF-8?q?march=C3=A9=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 10 ++ src/figures.py | 95 ++++++------- src/pages/observatoire.py | 274 +++++++++++++++++++++++++++++--------- 3 files changed, 271 insertions(+), 108 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 9705af4..12dd156 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -197,6 +197,16 @@ p.version > a { background-color: rgba(255, 240, 240, 0.4); } +#filters .col > * { + margin-bottom: 6px; +} + +#filters input[type="text"] { + border: 1px #ccc solid; + border-radius: 3px; + padding-left: 8px; +} + /* --- Tables (Dash & Custom) --- */ /* Table Menu (Exports etc) */ diff --git a/src/figures.py b/src/figures.py index c32f865..6f737cc 100644 --- a/src/figures.py +++ b/src/figures.py @@ -52,19 +52,18 @@ def get_yearly_statistics(statistics, today_str) -> html.Div: return html.Div(children=table, className="marches_table") -def get_barchart_sources(df_source: pl.DataFrame, type_date: str): - lf = df_source.lazy() +def get_barchart_sources(lff: pl.LazyFrame, type_date: str): labels = { "dateNotification": "notification", "datePublicationDonnees": "publication des données", } - lf = lf.select("uid", type_date, "sourceDataset") + lff = lff.select("uid", type_date, "sourceDataset") - lf = lf.unique("uid") + lff = lff.unique("uid") # Rassemblement des datasets Atexo pour ne pas surcharger le graphique - lf = lf.with_columns( + lff = lff.with_columns( pl.when(pl.col("sourceDataset").str.starts_with("atexo")) .then(pl.lit("plateformes atexo")) .otherwise(pl.col("sourceDataset")) @@ -72,34 +71,34 @@ def get_barchart_sources(df_source: pl.DataFrame, type_date: str): ) # Rassemblement des datasets AWS pour ne pas surcharger le graphique - lf = lf.with_columns( + lff = lff.with_columns( pl.when(pl.col("sourceDataset").str.contains(r"aws|marches\-publics.info")) .then(pl.lit("aws")) .otherwise(pl.col("sourceDataset")) .alias("sourceDataset") ) - lf = lf.with_columns(pl.col(type_date).dt.year().alias("annee")) - lf = lf.filter( + lff = lff.with_columns(pl.col(type_date).dt.year().alias("annee")) + lff = lff.filter( pl.col(type_date).is_not_null() & pl.col("annee").is_between(2019, 2025) ) - lf = lf.with_columns(pl.col(type_date).cast(pl.String).str.head(7)) - lf = ( - lf.group_by([type_date, "sourceDataset"]) + lff = lff.with_columns(pl.col(type_date).cast(pl.String).str.head(7)) + lff = ( + lff.group_by([type_date, "sourceDataset"]) .len() .sort(by=[type_date, "len"], descending=True) ) - # lf = lf.with_columns( + # lff = lff.with_columns( # pl.when(pl.col("sourceDataset").is_null()).then( # pl.lit("Source inconnue")).alias("sourceDataset") # ) - lf = lf.sort(by=["sourceDataset"], descending=False) - df: pl.DataFrame = lf.collect(engine="streaming") + lff = lff.sort(by=["sourceDataset"], descending=False) + dff: pl.DataFrame = lff.collect(engine="streaming") fig = px.bar( - df, + dff, x=type_date, y="len", color="sourceDataset", @@ -111,7 +110,9 @@ def get_barchart_sources(df_source: pl.DataFrame, type_date: str): }, ) - return fig + graph = dcc.Graph(figure=fig) + + return graph def get_sources_tables(source_path) -> html.Div: @@ -301,33 +302,24 @@ class DataTable(dash_table.DataTable): ) -def get_duplicate_matrix() -> html.Div: +def get_duplicate_matrix() -> dcc.Graph: """ Fonction développée avec l'aide de la LLM Euria d'Infomaniak. :return: """ - result_df = pl.read_parquet( + lff = pl.scan_parquet( "https://www.data.gouv.fr/api/1/datasets/r/a545bf6c-8b24-46ed-b49f-a32bf02eaffa" ).sort("sourceDataset") - result_df = result_df.select( - ["sourceDataset", "unique"] + sorted(result_df.columns[2:]) + lff = lff.select( + ["sourceDataset", "unique"] + sorted(lff.collect_schema().names()[2:]) ) - description = dcc.Markdown(""" - Ce graphique illustre les doublons de marchés publics entre sources, c'est-à-dire la proportion de marchés publiés par plus d'une source. Il s'appuie sur les identifiants `uid` qui sont pour chaque marché la concaténation du SIRET de l'acheteur et de l'identifiant interne du marché. - - **Comment lire ce graphique ?** - - On part des codes de sources de données en ordonnée. Ces jeux de données sont documentés dans [À propos](/a-propos#sources). - - La première colonne (**unique**) représente le pourcentage de marchés fournis par cette source qui sont uniquement disponibles dans cette source. Plus le rouge est foncé, plus important est le pourcentage. Donc, à l'inverse, plus le rouge est clair dans la première colonne, plus la source en ordonnée a des marchés en commun avec d'autres sources, et donc plus on trouvera sur la même ligne d'autres cases plus ou moins foncées qui indiqueront avec quelles autres sources cette source partage des marchés. - - Passez votre souris sur une case pour avoir les pourcentages exacts. À noter que ces statistiques sont produites avant le dédoublonnement qui a lieu avant la publication en Open Data et sur ce site.""") + dff = lff.collect() # Extract data - z_data = result_df.select(pl.all().exclude("sourceDataset")).fill_null(0).to_numpy() - x_labels = result_df.columns[1:] # columns after "sourceDataset" - y_labels = result_df["sourceDataset"].to_list() + z_data = dff.select(pl.all().exclude("sourceDataset")).fill_null(0).to_numpy() + x_labels = dff.columns[1:] # columns after "sourceDataset" + y_labels = dff["sourceDataset"].to_list() # Create heatmap fig = go.Figure( @@ -345,7 +337,7 @@ def get_duplicate_matrix() -> html.Div: hoverongaps=False, showscale=True, hovertemplate=( - "%{z:.0%} des marchés de %{y} sont également présents dans %{x}" + "%{z:.0%} des marchés présents dans %{y} sont également présents dans %{x}" ), ) ) @@ -364,13 +356,7 @@ def get_duplicate_matrix() -> html.Div: margin=dict(l=100, r=50, t=80, b=100), # Add margin for labels ) - return html.Div( - children=[ - html.H3("Doublons de marchés entre les sources"), - description, - dcc.Graph(figure=fig), - ] - ) + return dcc.Graph(figure=fig) def get_geographic_maps(dff: pl.DataFrame) -> list | None: @@ -624,20 +610,39 @@ def make_card( return card -def make_donut(lff: pl.LazyFrame, names_col): +def make_donut( + lff: pl.LazyFrame, + names_col, + per_uid: bool, + nulls="?", +): title = data_schema[names_col]["title"] lff = lff.rename({names_col: title}) lff = lff.select("uid", title) + + if per_uid: + lff = lff.group_by("uid").first() + lff = lff.group_by(title).len("Nombre") - lff = lff.with_columns(pl.col(title).replace(None, pl.lit("?"))) + lff = lff.with_columns(pl.col(title).replace(None, pl.lit(nulls))) + dff = lff.collect(engine="streaming") + dff = dff.with_columns( + pl.col("Nombre") + .map_elements(format_number, return_dtype=pl.String) + .alias("Nombre_fmt") + ) fig = px.pie( - lff.collect(engine="streaming"), + dff, values="Nombre", names=title, hole=0.4, color_discrete_sequence=px.colors.qualitative.Safe, + custom_data=["Nombre_fmt"], + ) + fig = fig.update_traces( + texttemplate="%{label}
%{percent}", + hovertemplate="%{label}
%{customdata[0]}", ) - fig = fig.update_traces(texttemplate="%{label}
%{percent}") fig = fig.update_layout(showlegend=False, font=dict(size=14)) graph = dcc.Graph(figure=fig) return graph diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index eb726f7..2b2cee8 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -3,9 +3,15 @@ from datetime import datetime, timedelta import dash_bootstrap_components as dbc import polars as pl import polars.selectors as cs -from dash import Input, Output, callback, dcc, html, register_page +from dash import ALL, Input, Output, callback, ctx, dcc, html, register_page -from src.figures import get_geographic_maps, make_card, make_donut +from src.figures import ( + get_barchart_sources, + get_duplicate_matrix, + get_geographic_maps, + make_card, + make_donut, +) from src.utils import ( departements, df, @@ -38,6 +44,29 @@ for code, obj in departements.items(): layout = [ dcc.Store(id="dashboard-filters"), dcc.Location(id="dashboard_url"), + dbc.Modal( + [ + dbc.ModalHeader(dbc.ModalTitle("Montants")), + dbc.ModalBody( + [ + dcc.Markdown( + """ +Les données saisies et publiées par les acheteurs comportent de nombreux montants farfelus qui sabotent les statistiques, au lieu de montants estimés avec rigueur. On parle de montant atteignant parfois les millions de milliards. Certains réutilisateurs mettent de côté ces marchés ou bien modifient les montants selon des règles fatalement arbitraires. J'ai fait le choix de ne quasiment pas modifier les données* afin de visibiliser le problème. + +Alors, on fait comment ? + +\* Les montants composés de plus de 11 chiffres, sans les décimales, [sont ramenés](https://github.com/ColinMaudry/decp-processing/blob/main/src/tasks/clean.py#L63-L71) à 12 311 111 111, un nombre qui reste très élevé et qui est facilement reconnaissable. +""" + ), + ] + ), + dbc.ModalFooter( + dbc.Button("Fermer", id="montant-modal-close", className="ms-auto") + ), + ], + id="montant-modal", + is_open=False, + ), html.Div( className="container-fluid", children=[ @@ -56,66 +85,109 @@ layout = [ children=[ html.H5("Période d'attribution"), dbc.Row( - dcc.Dropdown( - id="dashboard_year", - options=options_years, - placeholder="12 derniers mois", + dbc.Col( + dcc.Dropdown( + id="dashboard_year", + options=options_years, + placeholder="12 derniers mois", + ), ), ), html.H5("Acheteur"), dbc.Row( - dcc.Dropdown( - id="dashboard_acheteur_categorie", - options=get_enum_values_as_dict( - "acheteur_categorie" + dbc.Col( + dcc.Input( + id="dashboard_acheteur_id", + placeholder="SIRET", + style={"width": "100%"}, ), - placeholder="Catégorie", ), ), dbc.Row( - dcc.Dropdown( - id="dashboard_acheteur_departement_code", - searchable=True, - multi=True, - placeholder="Code département acheteur", - options=options_departements, + dbc.Col( + dcc.Dropdown( + id="dashboard_acheteur_categorie", + options=get_enum_values_as_dict( + "acheteur_categorie" + ), + placeholder="Catégorie", + ) + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_acheteur_departement_code", + searchable=True, + multi=True, + placeholder="Département", + options=options_departements, + ), ), ), html.H5("Titulaire"), dbc.Row( - dcc.Dropdown( - id="dashboard_titulaire_categorie", - placeholder="Catégorie", - options=get_enum_values_as_dict( - "titulaire_categorie" + dbc.Col( + dcc.Input( + id="dashboard_titulaire_id", + placeholder="SIRET", + style={"width": "100%"}, + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_titulaire_categorie", + placeholder="Catégorie", + options=get_enum_values_as_dict( + "titulaire_categorie" + ), + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_titulaire_departement_code", + searchable=True, + multi=True, + placeholder="Département", + options=options_departements, ), ), ), html.H5("Marché"), dbc.Row( - dcc.Dropdown( - id="dashboard_marche_type", - placeholder="Type", - options=get_enum_values_as_dict("type"), - ), - ), - dbc.Row( - dcc.Dropdown( - id="dashboard_marche_considerationsSociales", - placeholder="Considérations sociales", - options=get_enum_values_as_dict( - "considerationsSociales" + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_type", + placeholder="Type", + options=get_enum_values_as_dict("type"), ), - multi=True, ), ), dbc.Row( - dcc.Dropdown( - id="dashboard_marche_considerationsEnvironnementales", - placeholder="Considérations environnementales", - multi=True, - options=get_enum_values_as_dict( - "considerationsEnvironnementales" + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_considerationsSociales", + placeholder="Considérations sociales", + options=get_enum_values_as_dict( + "considerationsSociales" + ), + multi=True, + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_considerationsEnvironnementales", + placeholder="Considérations environnementales", + multi=True, + options=get_enum_values_as_dict( + "considerationsEnvironnementales" + ), ), ), ), @@ -140,21 +212,27 @@ layout = [ @callback( Output("cards", "children"), Input("dashboard_year", "value"), + Input("dashboard_acheteur_id", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_titulaire_id", "value"), Input("dashboard_titulaire_categorie", "value"), + Input("dashboard_titulaire_departement_code", "value"), Input("dashboard_marche_type", "value"), Input("dashboard_marche_considerationsSociales", "value"), Input("dashboard_marche_considerationsEnvironnementales", "value"), ) def udpate_dashboard_cards( dashboard_year, + dashboard_acheteur_id, dashboard_acheteur_categorie, dashboard_acheteur_departement_code, + dashboard_titulaire_id, dashboard_titulaire_categorie, + dashboard_titulaire_departement_code, dashboard_marche_type, - dashboard_marche_considerationsSociales, - dashboard_marche_considerationsEnvironnementales, + dashboard_marche_considerations_sociales, + dashboard_marche_considerations_environnementales, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -165,6 +243,8 @@ def udpate_dashboard_cards( "montant", "considerationsSociales", "considerationsEnvironnementales", + "sourceDataset", + "type", ) # Application des filtres @@ -180,40 +260,57 @@ def udpate_dashboard_cards( ## Acheteur - if dashboard_acheteur_categorie: - lff = lff.filter(pl.col("acheteur_categorie") == dashboard_acheteur_categorie) - - if dashboard_acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in( - dashboard_acheteur_departement_code + if dashboard_acheteur_id: + lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) + else: + if dashboard_acheteur_categorie: + lff = lff.filter( + pl.col("acheteur_categorie") == dashboard_acheteur_categorie + ) + + if dashboard_acheteur_departement_code: + lff = lff.filter( + pl.col("acheteur_departement_code").is_in( + dashboard_acheteur_departement_code + ) ) - ) ## Titulaire - if dashboard_titulaire_categorie: - lff = lff.filter(pl.col("titulaire_categorie") == dashboard_titulaire_categorie) + if dashboard_titulaire_id: + lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) + else: + if dashboard_titulaire_categorie: + lff = lff.filter( + pl.col("titulaire_categorie") == dashboard_titulaire_categorie + ) + + if dashboard_titulaire_departement_code: + lff = lff.filter( + pl.col("titulaire_departement_code").is_in( + dashboard_titulaire_departement_code + ) + ) ## Marché if dashboard_marche_type: lff = lff.filter(pl.col("type") == dashboard_marche_type) - if dashboard_marche_considerationsSociales: + if dashboard_marche_considerations_sociales: lff = lff.filter( pl.col("considerationsSociales") .str.split(", ") - .list.set_intersection(dashboard_marche_considerationsSociales) + .list.set_intersection(dashboard_marche_considerations_sociales) .list.len() > 0 ) - if dashboard_marche_considerationsEnvironnementales: + if dashboard_marche_considerations_environnementales: lff = lff.filter( pl.col("considerationsEnvironnementales") .str.split(", ") - .list.set_intersection(dashboard_marche_considerationsEnvironnementales) + .list.set_intersection(dashboard_marche_considerations_environnementales) .list.len() > 0 ) @@ -228,7 +325,8 @@ def udpate_dashboard_cards( df_per_uid = ( dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first()) ) - total_montant = df_per_uid.select(pl.col("montant").sum()).item() + + total_montant = int(df_per_uid.select(pl.col("montant").sum()).item()) nb_marches = df_per_uid.height cards = [] @@ -241,19 +339,69 @@ def udpate_dashboard_cards( html.P( ["Nombre de titulaires : ", html.Strong(str(format_number(nb_titulaires)))] ), - html.P(["Montant total : ", html.Strong(format_number(total_montant) + " €")]), + html.P( + [ + "Montant total (", + html.Span( + "?", + id={"type": "modal-trigger", "index": "montant"}, + style={"cursor": "pointer", "textDecoration": "underline dotted"}, + ), + ") : ", + html.Strong(format_number(total_montant) + " €"), + ] + ), ] cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) - donut_acheteur_categorie = make_donut(lff, "acheteur_categorie") - cards.append(make_card(title="Catégorie d'acheteur", fig=donut_acheteur_categorie)) + donut_acheteur_categorie = make_donut( + lff, "acheteur_categorie", nulls="Autres", per_uid=True + ) + cards.append( + make_card( + title="Catégorie d'acheteur", fig=donut_acheteur_categorie, lg=12, xl=8 + ) + ) - donut_titulaire_categorie = make_donut(lff, "titulaire_categorie") + donut_titulaire_categorie = make_donut( + lff, "titulaire_categorie", per_uid=False, nulls="?" + ) cards.append( make_card(title="Catégorie d'entreprise", fig=donut_titulaire_categorie) ) + donut_marche_type = make_donut(lff, "type", per_uid=True, nulls="?") + cards.append(make_card(title="Type d'achat", fig=donut_marche_type)) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) - return dbc.Row(children=cards + geographic_maps) + other_cards = [] + + sources_barchart = get_barchart_sources(lff, type_date="dateNotification") + other_cards.append( + make_card(title="Sources de données", fig=sources_barchart, lg=12, xl=8) + ) + + duplicate_matrix = get_duplicate_matrix() + other_cards.append( + make_card( + title="Matrice de doublons entre sources de données", + subtitle="Ce graphique illustre les doublons de marchés publics entre sources, c'est-à-dire la proportion de marchés publiés par plus d'une source.", + fig=duplicate_matrix, + lg=12, + xl=8, + ) + ) + + return dbc.Row(children=cards + geographic_maps + other_cards) + + +@callback( + Output("montant-modal", "is_open"), + Input({"type": "modal-trigger", "index": ALL}, "n_clicks"), + Input("montant-modal-close", "n_clicks"), + prevent_initial_call=True, +) +def toggle_montant_modal(n_triggers, _close): + return isinstance(ctx.triggered_id, dict) and any(n_triggers) From 79a06f996e46eed25ef1b4ad10a1112826d82eec Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 13:52:29 +0100 Subject: [PATCH 15/75] =?UTF-8?q?Bouton=20de=20t=C3=A9l=C3=A9chargement=20?= =?UTF-8?q?des=20donn=C3=A9es=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 1 - src/pages/observatoire.py | 224 ++++++++++++++++++++++++++------------ 2 files changed, 152 insertions(+), 73 deletions(-) diff --git a/src/figures.py b/src/figures.py index 6f737cc..8337f9c 100644 --- a/src/figures.py +++ b/src/figures.py @@ -102,7 +102,6 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): x=type_date, y="len", color="sourceDataset", - title=f"Nombre de marchés attribués par date de {labels[type_date]} et source de données", labels={ "len": "Nombre de marchés", type_date: f"Mois de {labels[type_date]}", diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 2b2cee8..d48518c 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta import dash_bootstrap_components as dbc import polars as pl import polars.selectors as cs -from dash import ALL, Input, Output, callback, ctx, dcc, html, register_page +from dash import ALL, Input, Output, State, callback, ctx, dcc, html, register_page from src.figures import ( get_barchart_sources, @@ -41,6 +41,70 @@ for code, obj in departements.items(): options_departements[code] = f"{obj['departement']} ({code})" +def _apply_filters( + lff: pl.LazyFrame, + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, +) -> pl.LazyFrame: + if year: + lff = lff.filter(pl.col("dateNotification").dt.year() == int(year)) + else: + lff = lff.filter( + pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) + ) + + if acheteur_id: + lff = lff.filter(pl.col("acheteur_id").str.contains(acheteur_id)) + else: + if acheteur_categorie: + lff = lff.filter(pl.col("acheteur_categorie") == acheteur_categorie) + if acheteur_departement_code: + lff = lff.filter( + pl.col("acheteur_departement_code").is_in(acheteur_departement_code) + ) + + if titulaire_id: + lff = lff.filter(pl.col("titulaire_id").str.contains(titulaire_id)) + else: + if titulaire_categorie: + lff = lff.filter(pl.col("titulaire_categorie") == titulaire_categorie) + if titulaire_departement_code: + lff = lff.filter( + pl.col("titulaire_departement_code").is_in(titulaire_departement_code) + ) + + if marche_type: + lff = lff.filter(pl.col("type") == marche_type) + + if considerations_sociales: + lff = lff.filter( + pl.col("considerationsSociales") + .str.split(", ") + .list.set_intersection(considerations_sociales) + .list.len() + > 0 + ) + + if considerations_environnementales: + lff = lff.filter( + pl.col("considerationsEnvironnementales") + .str.split(", ") + .list.set_intersection(considerations_environnementales) + .list.len() + > 0 + ) + + return lff + + layout = [ dcc.Store(id="dashboard-filters"), dcc.Location(id="dashboard_url"), @@ -55,7 +119,7 @@ Les données saisies et publiées par les acheteurs comportent de nombreux monta Alors, on fait comment ? -\* Les montants composés de plus de 11 chiffres, sans les décimales, [sont ramenés](https://github.com/ColinMaudry/decp-processing/blob/main/src/tasks/clean.py#L63-L71) à 12 311 111 111, un nombre qui reste très élevé et qui est facilement reconnaissable. +\\* Les montants composés de plus de 11 chiffres, sans les décimales, [sont ramenés](https://github.com/ColinMaudry/decp-processing/blob/main/src/tasks/clean.py#L63-L71) à 12 311 111 111, un nombre qui reste très élevé et qui est facilement reconnaissable. """ ), ] @@ -191,6 +255,13 @@ Alors, on fait comment ? ), ), ), + dcc.Download(id="download-observatoire"), + dbc.Button( + "Télécharger au format Excel", + id="btn-download-observatoire", + disabled=True, + className="mt-2", + ), ], ), dbc.Col( @@ -211,6 +282,8 @@ Alors, on fait comment ? @callback( Output("cards", "children"), + Output("btn-download-observatoire", "disabled"), + Output("btn-download-observatoire", "children"), Input("dashboard_year", "value"), Input("dashboard_acheteur_id", "value"), Input("dashboard_acheteur_categorie", "value"), @@ -246,74 +319,19 @@ def udpate_dashboard_cards( "sourceDataset", "type", ) - - # Application des filtres - - ## Période - - if dashboard_year: - lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) - else: - lff = lff.filter( - pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) - ) - - ## Acheteur - - if dashboard_acheteur_id: - lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) - else: - if dashboard_acheteur_categorie: - lff = lff.filter( - pl.col("acheteur_categorie") == dashboard_acheteur_categorie - ) - - if dashboard_acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in( - dashboard_acheteur_departement_code - ) - ) - - ## Titulaire - - if dashboard_titulaire_id: - lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) - else: - if dashboard_titulaire_categorie: - lff = lff.filter( - pl.col("titulaire_categorie") == dashboard_titulaire_categorie - ) - - if dashboard_titulaire_departement_code: - lff = lff.filter( - pl.col("titulaire_departement_code").is_in( - dashboard_titulaire_departement_code - ) - ) - - ## Marché - - if dashboard_marche_type: - lff = lff.filter(pl.col("type") == dashboard_marche_type) - - if dashboard_marche_considerations_sociales: - lff = lff.filter( - pl.col("considerationsSociales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_sociales) - .list.len() - > 0 - ) - - if dashboard_marche_considerations_environnementales: - lff = lff.filter( - pl.col("considerationsEnvironnementales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_environnementales) - .list.len() - > 0 - ) + lff = _apply_filters( + lff, + dashboard_year, + dashboard_acheteur_id, + dashboard_acheteur_categorie, + dashboard_acheteur_departement_code, + dashboard_titulaire_id, + dashboard_titulaire_categorie, + dashboard_titulaire_departement_code, + dashboard_marche_type, + dashboard_marche_considerations_sociales, + dashboard_marche_considerations_environnementales, + ) # Génération des métriques dff = lff.collect(engine="streaming") @@ -329,6 +347,13 @@ def udpate_dashboard_cards( total_montant = int(df_per_uid.select(pl.col("montant").sum()).item()) nb_marches = df_per_uid.height + if nb_marches == 0: + dl_disabled, dl_text = True, "Pas de données à télécharger" + elif nb_marches > 65000: + dl_disabled, dl_text = True, "Téléchargement désactivé au-delà de 65 000 lignes" + else: + dl_disabled, dl_text = False, "Télécharger au format Excel" + cards = [] card_basic_counts = [ @@ -380,7 +405,13 @@ def udpate_dashboard_cards( sources_barchart = get_barchart_sources(lff, type_date="dateNotification") other_cards.append( - make_card(title="Sources de données", fig=sources_barchart, lg=12, xl=8) + make_card( + title="Sources de données", + subtitle="Nombre de marchés attribués par mois de notification et source de données", + fig=sources_barchart, + lg=12, + xl=8, + ) ) duplicate_matrix = get_duplicate_matrix() @@ -394,7 +425,56 @@ def udpate_dashboard_cards( ) ) - return dbc.Row(children=cards + geographic_maps + other_cards) + return dbc.Row(children=cards + geographic_maps + other_cards), dl_disabled, dl_text + + +@callback( + Output("download-observatoire", "data"), + Input("btn-download-observatoire", "n_clicks"), + State("dashboard_year", "value"), + State("dashboard_acheteur_id", "value"), + State("dashboard_acheteur_categorie", "value"), + State("dashboard_acheteur_departement_code", "value"), + State("dashboard_titulaire_id", "value"), + State("dashboard_titulaire_categorie", "value"), + State("dashboard_titulaire_departement_code", "value"), + State("dashboard_marche_type", "value"), + State("dashboard_marche_considerationsSociales", "value"), + State("dashboard_marche_considerationsEnvironnementales", "value"), + prevent_initial_call=True, +) +def download_observatoire( + _n_clicks, + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, +): + lff = _apply_filters( + df.lazy(), + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, + ) + + def to_bytes(buffer): + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") + + date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes(to_bytes, filename=f"decp_observatoire_{date}.xlsx") @callback( From 3a70bbd9eaa3201b99de0c7a4836ed9691c1cd05 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:24:11 +0100 Subject: [PATCH 16/75] Ajout du spec : lien observatoire depuis recherche/tableau #65 Co-Authored-By: Claude Opus 4.6 --- ...18-observatoire-link-from-search-design.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md diff --git a/docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md b/docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md new file mode 100644 index 0000000..8843c0c --- /dev/null +++ b/docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md @@ -0,0 +1,78 @@ +# Observatoire Link from Search & Tableau Results + +## Problem + +Users searching for an organization (acheteur or titulaire) on the search page or browsing the tableau cannot jump directly to the observatoire page filtered for that organization. They must manually navigate and re-enter the identifier. + +## Solution + +Extend `add_links()` in `src/utils.py` to append an observatoire link (📊 emoji) to `_nom` columns, and add bidirectional URL parameter sync to the observatoire page. + +## Changes + +### 1. `src/utils.py` — `add_links()` modification + +The existing `add_links()` loop iterates over `["uid", "acheteur_nom", "titulaire_nom", "acheteur_id", "titulaire_id"]`. The `if col.startswith("acheteur_")` and `if col.startswith("titulaire_")` blocks match both `_nom` and `_id` columns. The observatoire link must only be appended to `_nom` columns, so it must be gated on `col == "acheteur_nom"` or `col == "titulaire_nom"` explicitly. + +For `acheteur_nom`, append an observatoire link after the existing detail page link: + +``` +Before: Ville de Paris +After: Ville de Paris 📊 +``` + +For `titulaire_nom`, same pattern but only when the existing `typeIdentifiant` guard passes (SIRET or null): + +``` +Before: Entreprise X +After: Entreprise X 📊 +``` + +The identifier used in the observatoire link (`acheteur_id` / `titulaire_id`) is the same `pl.col("acheteur_id")` / `pl.col("titulaire_id")` column value already used for the detail page link. + +The `_id` and `uid` columns are unchanged. + +### 2. `src/pages/observatoire.py` — URL parameter handling + +#### Callback A: URL → Inputs (page load) + +- Trigger: `Input("dashboard_url", "search")` +- Outputs: `Output("dashboard_acheteur_id", "value")`, `Output("dashboard_titulaire_id", "value")`, `Output("dashboard_url", "search")` (to clear it) +- `prevent_initial_call=False` (must fire on page load to read URL params) +- If `search` is empty or None: return `no_update` for all outputs +- Otherwise: parse query params with `urllib.parse.parse_qs` +- Set `dashboard_acheteur_id` from `?acheteur_id=` param, or `no_update` if absent +- Set `dashboard_titulaire_id` from `?titulaire_id=` param, or `no_update` if absent +- Return `""` for `dashboard_url.search` to clear the URL and prevent re-triggering +- No validation of param values — consistent with existing input handling in the observatoire callbacks + +#### Callback B: Inputs → shareable URL + +- Trigger: `Input("dashboard_acheteur_id", "value")`, `Input("dashboard_titulaire_id", "value")` +- State: `State("dashboard_url", "href")` for base URL +- `prevent_initial_call=True` (avoid generating URL on initial empty state) +- Build query string with `urllib.parse.urlencode`, omitting empty values +- Write full URL to a new `share-url` input component +- Render a `dcc.Clipboard` + share button (same pattern as tableau.py) + +#### Callback chain + +When navigating from search with `?acheteur_id=123`: Callback A fires on page load, sets input values, clears URL search. The input value changes then trigger both the existing `udpate_dashboard_cards` callback and Callback B. Dash handles this chaining deterministically — no race condition. + +#### Layout additions + +- A `dcc.Input(id="share-url", ...)` (hidden or read-only) to hold the shareable URL +- A `dcc.Clipboard` share/copy button near the filters + +### 3. Reuse of existing `dcc.Location` + +The existing `dcc.Location(id="dashboard_url")` component is reused — no new Location component needed. + +## Future extension + +The bidirectional URL sync pattern is designed to extend to all observatoire filters (year, categories, departments, market type, etc.) by adding more params to both callbacks. + +## Files touched + +- `src/utils.py` — modify `add_links()` +- `src/pages/observatoire.py` — add 2 callbacks, add share-url + clipboard to layout From d1a876ba9cfd17a571e071ed6bb54580afd1cc10 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:35:58 +0100 Subject: [PATCH 17/75] =?UTF-8?q?Plan=20d'impl=C3=A9mentation=20:=20lien?= =?UTF-8?q?=20observatoire=20depuis=20recherche/tableau=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- .../plans/2026-03-18-observatoire-link.md | 473 ++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-18-observatoire-link.md diff --git a/docs/superpowers/plans/2026-03-18-observatoire-link.md b/docs/superpowers/plans/2026-03-18-observatoire-link.md new file mode 100644 index 0000000..f81a2d4 --- /dev/null +++ b/docs/superpowers/plans/2026-03-18-observatoire-link.md @@ -0,0 +1,473 @@ +# Observatoire Link from Search & Tableau Results — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let users jump from search/tableau results to the observatoire page, pre-filtered for a given organization, via a 📊 link in the `_nom` columns. + +**Architecture:** Modify `add_links()` in `src/utils.py` to append an observatoire link to `_nom` columns. Add two callbacks to `src/pages/observatoire.py` for bidirectional URL ↔ filter sync using the existing `dcc.Location(id="dashboard_url")`. Add a share URL input and clipboard button to the observatoire layout. + +**Tech Stack:** Dash 3.4, Polars, `urllib.parse`, `dcc.Location`, `dcc.Clipboard` + +**Spec:** `docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md` + +--- + +### Task 1: Add observatoire link to `acheteur_nom` in `add_links()` + +**Files:** + +- Modify: `src/utils.py:82-91` (the `acheteur_` block inside `add_links()`) +- Test: `tests/test_main.py` + +**Context:** The `add_links()` function loops over column names. The `if col.startswith("acheteur_")` block (lines 82-91) currently wraps both `acheteur_nom` and `acheteur_id` in a detail page link. We must only append the observatoire link when `col == "acheteur_nom"`. + +- [ ] **Step 1: Write a unit test for the observatoire link in acheteur_nom** + +In `tests/test_main.py`, add a test that calls `add_links()` on a minimal DataFrame and checks the `acheteur_nom` column contains both the detail link and the observatoire link, while `acheteur_id` does NOT contain the observatoire link. + +```python +def test_004_add_links_observatoire_acheteur(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "acheteur_id": ["a1"], + "acheteur_nom": ["ACHETEUR 1"], + } + ) + result = add_links(dff) + nom_value = result["acheteur_nom"][0] + id_value = result["acheteur_id"][0] + + # acheteur_nom should contain detail link + observatoire link + assert "/acheteurs/a1" in nom_value + assert "ACHETEUR 1" in nom_value + assert '/observatoire?acheteur_id=a1' in nom_value + assert "📊" in nom_value + + # acheteur_id should NOT contain observatoire link + assert "/observatoire" not in id_value +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_004_add_links_observatoire_acheteur -v` +Expected: FAIL — `'/observatoire?acheteur_id=a1'` not found in the output string. + +- [ ] **Step 3: Implement the observatoire link for acheteur_nom** + +In `src/utils.py`, modify the `if col.startswith("acheteur_")` block (lines 82-91). Gate the observatoire link append on `col == "acheteur_nom"`: + +```python + if col.startswith("acheteur_"): + detail_link = ( + '' + + pl.col(col) + + "" + ) + if col == "acheteur_nom": + detail_link = ( + detail_link + + ' 📊' + ) + dff = dff.with_columns(detail_link.alias(col)) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_004_add_links_observatoire_acheteur -v` +Expected: PASS + +- [ ] **Step 5: Update `test_001` to account for the new emoji in cell text** + +The existing `test_001` asserts `result_table.find_element(...).text == name` for `acheteur_nom`. The cell text now includes "📊" from the observatoire link. Update the assertion in `tests/test_main.py` to use `startswith` instead of exact match: + +```python + assert result_table.find_element( + by=By.CSS_SELECTOR, value=f'td[data-dash-column="{org_type}_nom"]' + ).text.startswith( + name + ), f"The search result should have the right {org_type} name" +``` + +- [ ] **Step 6: Run `test_001` to verify it still passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_001_logo_and_search -v` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/utils.py tests/test_main.py +git commit -m "Ajout du lien observatoire dans acheteur_nom via add_links() #65" +``` + +--- + +### Task 2: Add observatoire link to `titulaire_nom` in `add_links()` + +**Files:** + +- Modify: `src/utils.py:64-81` (the `titulaire_` block inside `add_links()`) +- Test: `tests/test_main.py` + +**Context:** The `titulaire_` block (lines 64-81) uses a `pl.when().then().otherwise()` pattern because it guards on `titulaire_typeIdentifiant` being SIRET or null. The observatoire link must be appended inside the `.then()` branch, and only when `col == "titulaire_nom"`. Note: this block requires `titulaire_typeIdentifiant` to be present in the DataFrame. + +- [ ] **Step 1: Write a unit test for the observatoire link in titulaire_nom** + +```python +def test_005_add_links_observatoire_titulaire(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "titulaire_id": ["t1"], + "titulaire_nom": ["TITULAIRE 1"], + "titulaire_typeIdentifiant": ["SIRET"], + } + ) + result = add_links(dff) + nom_value = result["titulaire_nom"][0] + id_value = result["titulaire_id"][0] + + # titulaire_nom should contain detail link + observatoire link + assert "/titulaires/t1" in nom_value + assert "TITULAIRE 1" in nom_value + assert '/observatoire?titulaire_id=t1' in nom_value + assert "📊" in nom_value + + # titulaire_id should NOT contain observatoire link + assert "/observatoire" not in id_value +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_005_add_links_observatoire_titulaire -v` +Expected: FAIL — `'/observatoire?titulaire_id=t1'` not found. + +- [ ] **Step 3: Implement the observatoire link for titulaire_nom** + +In `src/utils.py`, modify the `if col.startswith("titulaire_")` block (lines 64-81). The `.then()` branch must build the link differently when `col == "titulaire_nom"`: + +```python + if col.startswith("titulaire_"): + detail_link = ( + '' + + pl.col(col) + + "" + ) + if col == "titulaire_nom": + detail_link = ( + detail_link + + ' 📊' + ) + dff = dff.with_columns( + pl.when( + pl.Expr.or_( + pl.col("titulaire_typeIdentifiant").is_null(), + pl.col("titulaire_typeIdentifiant") == "SIRET", + ) + ) + .then(detail_link) + .otherwise(pl.col(col)) + .alias(col) + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_005_add_links_observatoire_titulaire -v` +Expected: PASS + +- [ ] **Step 5: Run all tests so far to check for regressions** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_004_add_links_observatoire_acheteur tests/test_main.py::test_005_add_links_observatoire_titulaire -v` +Expected: both PASS + +- [ ] **Step 6: Commit** + +```bash +git add src/utils.py tests/test_main.py +git commit -m "Ajout du lien observatoire dans titulaire_nom via add_links() #65" +``` + +--- + +### Task 3: Observatoire Callback A — URL → Inputs (page load) + +**Files:** + +- Modify: `src/pages/observatoire.py` (add import + new callback after line 281) +- Test: `tests/test_main.py` + +**Context:** The existing `dcc.Location(id="dashboard_url")` is in the observatoire layout. A new callback reads `dashboard_url.search` on page load, parses query params, and sets `dashboard_acheteur_id.value` and/or `dashboard_titulaire_id.value`. It also clears `dashboard_url.search` to `""` to prevent re-triggering. Two imports must be added: `import urllib.parse` at the top of the file, and `no_update` to the existing `from dash import ...` line (currently: `from dash import ALL, Input, Output, State, callback, ctx, dcc, html, register_page` — add `no_update` to this). + +- [ ] **Step 1: Write a Selenium test for URL → Input sync** + +This test navigates to `/observatoire?acheteur_id=a1` and verifies the SIRET input gets populated. + +```python +def test_006_observatoire_url_to_input(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + dash_duo.wait_for_text_to_equal( + "#dashboard_acheteur_id", "", timeout=4 + ) # Wait for callback + import time + time.sleep(1) # Allow callback chain to complete + + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated from URL param" + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_006_observatoire_url_to_input -v` +Expected: FAIL — the input value is empty because no callback reads URL params yet. + +- [ ] **Step 3: Implement Callback A** + +Add `import urllib.parse` to the imports at the top of `src/pages/observatoire.py` (after line 1). Also add `no_update` to the existing dash import line: + +```python +from dash import ALL, Input, Output, State, callback, ctx, dcc, html, no_update, register_page +``` + +Add the callback after the `layout` list ends, before existing callbacks: + +```python +@callback( + Output("dashboard_acheteur_id", "value"), + Output("dashboard_titulaire_id", "value"), + Output("dashboard_url", "search"), + Input("dashboard_url", "search"), +) +def restore_filters_from_url(search): + if not search: + return no_update, no_update, no_update + + params = urllib.parse.parse_qs(search.lstrip("?")) + + acheteur_id = params.get("acheteur_id", [None])[0] or no_update + titulaire_id = params.get("titulaire_id", [None])[0] or no_update + + return acheteur_id, titulaire_id, "" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_006_observatoire_url_to_input -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/pages/observatoire.py tests/test_main.py +git commit -m "Callback URL → filtres sur la page observatoire #65" +``` + +--- + +### Task 4: Observatoire Callback B — Inputs → shareable URL + layout + +**Files:** + +- Modify: `src/pages/observatoire.py` (add layout components + new callback) +- Test: `tests/test_main.py` + +**Context:** Following the tableau.py pattern (lines 237-238 for layout, lines 399-450 for callback), add a hidden `share-url` input and a `copy-container` div to the observatoire layout. The callback listens to the ID inputs and builds a shareable URL. Component IDs must be unique across the app, so use `observatoire-share-url` and `observatoire-copy-container` to avoid collisions with tableau's `share-url` and `copy-container`. + +- [ ] **Step 1: Write a test for the shareable URL generation** + +```python +def test_007_observatoire_share_url(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#observatoire-share-url", timeout=4) + + import time + time.sleep(1) # Allow callback chain to complete + + share_url_input = dash_duo.find_element("#observatoire-share-url") + share_url_value = share_url_input.get_attribute("value") + + assert "acheteur_id=a1" in share_url_value, ( + f"Share URL should contain acheteur_id param, got: {share_url_value}" + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_007_observatoire_share_url -v` +Expected: FAIL — `#observatoire-share-url` element does not exist yet. + +- [ ] **Step 3: Add layout components to observatoire** + +In `src/pages/observatoire.py`, add the share URL input and copy container inside the filters column (after the download button, before the closing `]` of the `id="filters"` children list, around line 264): + +```python + dcc.Input( + id="observatoire-share-url", + readOnly=True, + style={"display": "none"}, + ), + html.Div(id="observatoire-copy-container"), +``` + +- [ ] **Step 4: Implement Callback B** + +Add after Callback A in `src/pages/observatoire.py`: + +```python +@callback( + Output("observatoire-share-url", "value"), + Output("observatoire-copy-container", "children"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_titulaire_id", "value"), + State("dashboard_url", "href"), + prevent_initial_call=True, +) +def sync_observatoire_share_url(acheteur_id, titulaire_id, href): + if not href: + return no_update, no_update + + base_url = href.split("?")[0] + + params = {} + if acheteur_id: + params["acheteur_id"] = acheteur_id + if titulaire_id: + params["titulaire_id"] = titulaire_id + + query_string = urllib.parse.urlencode(params) + full_url = f"{base_url}?{query_string}" if query_string else base_url + + copy_button = dcc.Clipboard( + id="btn-copy-observatoire-url", + target_id="observatoire-share-url", + title="Copier l'URL de cette vue", + style={ + "display": "inline-block", + "fontSize": 20, + "verticalAlign": "top", + "cursor": "pointer", + }, + className="fa fa-link", + children=[ + dbc.Button( + "Partager", + className="btn btn-primary mt-2", + title="Copier l'adresse de cette vue filtrée pour la partager.", + ) + ], + ) + + return full_url, copy_button +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_007_observatoire_share_url -v` +Expected: PASS + +- [ ] **Step 6: Run all tests to check for regressions** + +Run: `source .venv/bin/activate && pytest tests/test_main.py -v` +Expected: all tests PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/pages/observatoire.py tests/test_main.py +git commit -m "URL partageable pour la page observatoire #65" +``` + +--- + +### Task 5: End-to-end integration test + +**Files:** + +- Test: `tests/test_main.py` + +**Context:** Verify the full flow: search for an organization on the homepage, see the 📊 link in results, click it, arrive on the observatoire with the correct input populated. + +- [ ] **Step 1: Write end-to-end test** + +```python +def test_008_search_to_observatoire(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Search for an acheteur + search_bar = dash_duo.find_element("#search") + search_bar.send_keys("ACHETEUR 1") + search_bar.send_keys(Keys.ENTER) + + dash_duo.wait_for_element("#results_acheteur_datatable", timeout=2) + + # Find the observatoire link in acheteur_nom column + observatoire_link = dash_duo.find_element( + '#results_acheteur_datatable td[data-dash-column="acheteur_nom"] a[href*="observatoire"]' + ) + assert "📊" in observatoire_link.text + + # Click the observatoire link + observatoire_link.click() + + # Wait for observatoire page to load + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + import time + time.sleep(1) # Allow callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated after navigating from search" + ) +``` + +- [ ] **Step 2: Run end-to-end test** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_008_search_to_observatoire -v` +Expected: PASS + +- [ ] **Step 3: Run the full test suite** + +Run: `source .venv/bin/activate && pytest tests/test_main.py -v` +Expected: all tests PASS + +- [ ] **Step 4: Commit** + +```bash +git add tests/test_main.py +git commit -m "Test e2e : recherche → observatoire #65" +``` From 1d88682f85394fb478c8708f9f708317e8277318 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:43:10 +0100 Subject: [PATCH 18/75] Ajout du lien observatoire dans acheteur_nom via add_links() #65 Co-Authored-By: Claude Sonnet 4.6 --- src/utils.py | 24 +++++++++++++++--------- tests/test_main.py | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/utils.py b/src/utils.py index 642bad4..09eca00 100644 --- a/src/utils.py +++ b/src/utils.py @@ -80,15 +80,21 @@ def add_links(dff: pl.DataFrame): .alias(col) ) if col.startswith("acheteur_"): - dff = dff.with_columns( - ( - '' - + pl.col(col) - + "" - ).alias(col) + detail_link = ( + '' + + pl.col(col) + + "" ) + if col == "acheteur_nom": + detail_link = ( + detail_link + + ' 📊' + ) + dff = dff.with_columns(detail_link.alias(col)) if col == "uid": dff = dff.with_columns( ( @@ -648,7 +654,7 @@ def prepare_table_data( # Remplace les strings null par "", mais pas les numeric null dff = dff.fill_null("") - # Ajout des liens vers l'annuaire des entreprises + # Ajout des liens vers les pages de détails dff = add_links(dff) # Ajout des liens vers les fichiers Open Data diff --git a/tests/test_main.py b/tests/test_main.py index 1b8e7db..39b38a2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -29,12 +29,11 @@ def test_001_logo_and_search(dash_duo: DashComposite): assert len(result_table.find_elements(by=By.TAG_NAME, value="tr")) == 2, ( "The search should return only one result" ) # header row + 1 result - assert ( - result_table.find_element( - by=By.CSS_SELECTOR, value=f'td[data-dash-column="{org_type}_nom"]' - ).text - == name - ), f"The search result should have the right {org_type} name" + assert result_table.find_element( + by=By.CSS_SELECTOR, value=f'td[data-dash-column="{org_type}_nom"]' + ).text.startswith(name), ( + f"The search result should have the right {org_type} name" + ) def test_002_filter_persistence(dash_duo: DashComposite): @@ -87,3 +86,28 @@ def test_003_tableau_download(dash_duo: DashComposite): ) assert output["type"] is None assert output["base64"] is True + + +def test_004_add_links_observatoire_acheteur(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "acheteur_id": ["a1"], + "acheteur_nom": ["ACHETEUR 1"], + } + ) + result = add_links(dff) + nom_value = result["acheteur_nom"][0] + id_value = result["acheteur_id"][0] + + # acheteur_nom should contain detail link + observatoire link + assert "/acheteurs/a1" in nom_value + assert "ACHETEUR 1" in nom_value + assert "/observatoire?acheteur_id=a1" in nom_value + assert "📊" in nom_value + + # acheteur_id should NOT contain observatoire link + assert "/observatoire" not in id_value From bf1791635f45d809980e84ae0e5b21299dc72ccd Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:48:05 +0100 Subject: [PATCH 19/75] Ajout du lien observatoire dans titulaire_nom via add_links() #65 Co-Authored-By: Claude Sonnet 4.6 --- src/utils.py | 22 +++++++++++++++------- tests/test_main.py | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/utils.py b/src/utils.py index 09eca00..d791cf9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -62,6 +62,20 @@ def add_links(dff: pl.DataFrame): for col in ["uid", "acheteur_nom", "titulaire_nom", "acheteur_id", "titulaire_id"]: if col in dff.columns: if col.startswith("titulaire_"): + detail_link = ( + '' + + pl.col(col) + + "" + ) + if col == "titulaire_nom": + detail_link = ( + detail_link + + ' 📊' + ) dff = dff.with_columns( pl.when( pl.Expr.or_( @@ -69,13 +83,7 @@ def add_links(dff: pl.DataFrame): pl.col("titulaire_typeIdentifiant") == "SIRET", ) ) - .then( - '' - + pl.col(col) - + "" - ) + .then(detail_link) .otherwise(pl.col(col)) .alias(col) ) diff --git a/tests/test_main.py b/tests/test_main.py index 39b38a2..e3f77bb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -111,3 +111,29 @@ def test_004_add_links_observatoire_acheteur(): # acheteur_id should NOT contain observatoire link assert "/observatoire" not in id_value + + +def test_005_add_links_observatoire_titulaire(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "titulaire_id": ["t1"], + "titulaire_nom": ["TITULAIRE 1"], + "titulaire_typeIdentifiant": ["SIRET"], + } + ) + result = add_links(dff) + nom_value = result["titulaire_nom"][0] + id_value = result["titulaire_id"][0] + + # titulaire_nom should contain detail link + observatoire link + assert "/titulaires/t1" in nom_value + assert "TITULAIRE 1" in nom_value + assert "/observatoire?titulaire_id=t1" in nom_value + assert "📊" in nom_value + + # titulaire_id should NOT contain observatoire link + assert "/observatoire" not in id_value From 3745f6df74c50ef5371849ccac6b2913b4c5a3b4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:55:41 +0100 Subject: [PATCH 20/75] =?UTF-8?q?Callback=20URL=20=E2=86=92=20filtres=20su?= =?UTF-8?q?r=20la=20page=20observatoire=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 34 ++++++++++++++++++++++++++++++++-- tests/conftest.py | 5 +++++ tests/test_main.py | 20 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index d48518c..67ab659 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -1,9 +1,21 @@ +import urllib.parse from datetime import datetime, timedelta import dash_bootstrap_components as dbc import polars as pl import polars.selectors as cs -from dash import ALL, Input, Output, State, callback, ctx, dcc, html, register_page +from dash import ( + ALL, + Input, + Output, + State, + callback, + ctx, + dcc, + html, + no_update, + register_page, +) from src.figures import ( get_barchart_sources, @@ -107,7 +119,7 @@ def _apply_filters( layout = [ dcc.Store(id="dashboard-filters"), - dcc.Location(id="dashboard_url"), + dcc.Location(id="dashboard_url", refresh="callback-nav"), dbc.Modal( [ dbc.ModalHeader(dbc.ModalTitle("Montants")), @@ -280,6 +292,24 @@ Alors, on fait comment ? ] +@callback( + Output("dashboard_acheteur_id", "value"), + Output("dashboard_titulaire_id", "value"), + Output("dashboard_url", "search"), + Input("dashboard_url", "search"), +) +def restore_filters_from_url(search): + if not search: + return no_update, no_update, no_update + + params = urllib.parse.parse_qs(search.lstrip("?")) + + acheteur_id = params.get("acheteur_id", [None])[0] or no_update + titulaire_id = params.get("titulaire_id", [None])[0] or no_update + + return acheteur_id, titulaire_id, "" + + @callback( Output("cards", "children"), Output("btn-download-observatoire", "disabled"), diff --git a/tests/conftest.py b/tests/conftest.py index b14e0f8..f0f2b00 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,11 @@ def test_data(): "sourceFile": "test.xml", "sourceDataset": "test_dataset", "datePublicationDonnees": datetime.date(2025, 1, 1), + "considerationsSociales": "", + "considerationsEnvironnementales": "", + "type": "Marché", + "acheteur_categorie": "Collectivité", + "titulaire_categorie": "PME", } ] path = "tests/test.parquet" diff --git a/tests/test_main.py b/tests/test_main.py index e3f77bb..995a669 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -137,3 +137,23 @@ def test_005_add_links_observatoire_titulaire(): # titulaire_id should NOT contain observatoire link assert "/observatoire" not in id_value + + +def test_006_observatoire_url_to_input(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + import time + + time.sleep(1) # Allow callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated from URL param" + ) From e804b6bca2291d1ac3dff9208cd8fd1dc9ba6802 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:59:32 +0100 Subject: [PATCH 21/75] URL partageable pour la page observatoire #65 Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 52 +++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 22 +++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 67ab659..b42a99a 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -274,6 +274,12 @@ Alors, on fait comment ? disabled=True, className="mt-2", ), + dcc.Input( + id="observatoire-share-url", + readOnly=True, + style={"display": "none"}, + ), + html.Div(id="observatoire-copy-container"), ], ), dbc.Col( @@ -310,6 +316,52 @@ def restore_filters_from_url(search): return acheteur_id, titulaire_id, "" +@callback( + Output("observatoire-share-url", "value"), + Output("observatoire-copy-container", "children"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_titulaire_id", "value"), + State("dashboard_url", "href"), + prevent_initial_call=True, +) +def sync_observatoire_share_url(acheteur_id, titulaire_id, href): + if not href: + return no_update, no_update + + base_url = href.split("?")[0] + + params = {} + if acheteur_id: + params["acheteur_id"] = acheteur_id + if titulaire_id: + params["titulaire_id"] = titulaire_id + + query_string = urllib.parse.urlencode(params) + full_url = f"{base_url}?{query_string}" if query_string else base_url + + copy_button = dcc.Clipboard( + id="btn-copy-observatoire-url", + target_id="observatoire-share-url", + title="Copier l'URL de cette vue", + style={ + "display": "inline-block", + "fontSize": 20, + "verticalAlign": "top", + "cursor": "pointer", + }, + className="fa fa-link", + children=[ + dbc.Button( + "Partager", + className="btn btn-primary mt-2", + title="Copier l'adresse de cette vue filtrée pour la partager.", + ) + ], + ) + + return full_url, copy_button + + @callback( Output("cards", "children"), Output("btn-download-observatoire", "disabled"), diff --git a/tests/test_main.py b/tests/test_main.py index 995a669..6cd2903 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -157,3 +157,25 @@ def test_006_observatoire_url_to_input(dash_duo: DashComposite): assert acheteur_input.get_attribute("value") == "a1", ( "acheteur_id input should be populated from URL param" ) + + +def test_007_observatoire_share_url(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#observatoire-share-url", timeout=4) + + import time + + time.sleep(1) # Allow callback chain to complete + + share_url_input = dash_duo.find_element("#observatoire-share-url") + share_url_value = share_url_input.get_attribute("value") + + assert "acheteur_id=a1" in share_url_value, ( + f"Share URL should contain acheteur_id param, got: {share_url_value}" + ) From acb8500dc0dece7a13536e093e79848201dc72fd Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 15:02:27 +0100 Subject: [PATCH 22/75] =?UTF-8?q?Test=20e2e=20:=20recherche=20=E2=86=92=20?= =?UTF-8?q?observatoire=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- tests/test_main.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 6cd2903..6809636 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -179,3 +179,38 @@ def test_007_observatoire_share_url(dash_duo: DashComposite): assert "acheteur_id=a1" in share_url_value, ( f"Share URL should contain acheteur_id param, got: {share_url_value}" ) + + +def test_008_search_to_observatoire(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Search for an acheteur + search_bar = dash_duo.find_element("#search") + search_bar.send_keys("ACHETEUR 1") + search_bar.send_keys(Keys.ENTER) + + dash_duo.wait_for_element("#results_acheteur_datatable", timeout=2) + + # Find the observatoire link in acheteur_nom column + observatoire_link = dash_duo.find_element( + '#results_acheteur_datatable td[data-dash-column="acheteur_nom"] a[href*="observatoire"]' + ) + assert "📊" in observatoire_link.text + + # Click the observatoire link + observatoire_link.click() + + # Wait for observatoire page to load + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + import time + + time.sleep(1) # Allow callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated after navigating from search" + ) From 958c3956ea8ee57750e747d2d4cd8f1834819081 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 16:05:29 +0100 Subject: [PATCH 23/75] =?UTF-8?q?Correction=20des=20probl=C3=A8mes=20de=20?= =?UTF-8?q?double=20reload=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index b42a99a..6887f93 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -301,19 +301,25 @@ Alors, on fait comment ? @callback( Output("dashboard_acheteur_id", "value"), Output("dashboard_titulaire_id", "value"), - Output("dashboard_url", "search"), + # Output("dashboard_url", "search"), Input("dashboard_url", "search"), ) def restore_filters_from_url(search): if not search: - return no_update, no_update, no_update + return no_update, no_update params = urllib.parse.parse_qs(search.lstrip("?")) - acheteur_id = params.get("acheteur_id", [None])[0] or no_update - titulaire_id = params.get("titulaire_id", [None])[0] or no_update + def get_param_value(key): + values = params.get(key) + if values and values[0] is not None: + return values[0] + return no_update - return acheteur_id, titulaire_id, "" + acheteur_id = get_param_value("acheteur_id") + titulaire_id = get_param_value("titulaire_id") + + return acheteur_id, titulaire_id @callback( From 5d4c0b8438e1e6f507995a2bd047ba68124dc8c5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:31:45 +0100 Subject: [PATCH 24/75] test: failing test for observatoire localStorage filter persistence #65 --- tests/test_main.py | 75 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 6809636..f9d36b7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -51,7 +51,7 @@ def test_002_filter_persistence(dash_duo: DashComposite): _filter_input: WebElement = dash_duo.find_element(filter_input_selector) return _filter_input - for page in ["tableau", "acheteurs/a1", "titulaires/t1"]: + for page in ["tableau", "acheteurs/123", "titulaires/345"]: print("page:", page) filter_input = open_page_and_check_filter_input() filter_input.send_keys("11") # a UID that doesn't exist @@ -73,8 +73,8 @@ def test_003_tableau_download(dash_duo: DashComposite): outputs = [ download_data(1, "", [], None), - download_acheteur_data(1, dicts, "a1", "2025"), - download_titulaire_data(1, dicts, "t1", "2025"), + download_acheteur_data(1, dicts, "123", "2025"), + download_titulaire_data(1, dicts, "345", "2025"), ] for output in outputs: assert isinstance(output, dict) @@ -95,7 +95,7 @@ def test_004_add_links_observatoire_acheteur(): dff = pl.DataFrame( { - "acheteur_id": ["a1"], + "acheteur_id": ["123"], "acheteur_nom": ["ACHETEUR 1"], } ) @@ -104,9 +104,9 @@ def test_004_add_links_observatoire_acheteur(): id_value = result["acheteur_id"][0] # acheteur_nom should contain detail link + observatoire link - assert "/acheteurs/a1" in nom_value + assert "/acheteurs/123" in nom_value assert "ACHETEUR 1" in nom_value - assert "/observatoire?acheteur_id=a1" in nom_value + assert "/observatoire?acheteur_id=123" in nom_value assert "📊" in nom_value # acheteur_id should NOT contain observatoire link @@ -120,7 +120,7 @@ def test_005_add_links_observatoire_titulaire(): dff = pl.DataFrame( { - "titulaire_id": ["t1"], + "titulaire_id": ["345"], "titulaire_nom": ["TITULAIRE 1"], "titulaire_typeIdentifiant": ["SIRET"], } @@ -130,9 +130,9 @@ def test_005_add_links_observatoire_titulaire(): id_value = result["titulaire_id"][0] # titulaire_nom should contain detail link + observatoire link - assert "/titulaires/t1" in nom_value + assert "/titulaires/345" in nom_value assert "TITULAIRE 1" in nom_value - assert "/observatoire?titulaire_id=t1" in nom_value + assert "/observatoire?titulaire_id=345" in nom_value assert "📊" in nom_value # titulaire_id should NOT contain observatoire link @@ -146,7 +146,7 @@ def test_006_observatoire_url_to_input(dash_duo: DashComposite): dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) # Navigate to observatoire with acheteur_id query param - dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=123") dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) import time @@ -154,7 +154,7 @@ def test_006_observatoire_url_to_input(dash_duo: DashComposite): time.sleep(1) # Allow callback chain to complete acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") - assert acheteur_input.get_attribute("value") == "a1", ( + assert acheteur_input.get_attribute("value") == "123", ( "acheteur_id input should be populated from URL param" ) @@ -166,7 +166,7 @@ def test_007_observatoire_share_url(dash_duo: DashComposite): dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) # Navigate to observatoire with acheteur_id query param - dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=123") dash_duo.wait_for_element("#observatoire-share-url", timeout=4) import time @@ -176,7 +176,7 @@ def test_007_observatoire_share_url(dash_duo: DashComposite): share_url_input = dash_duo.find_element("#observatoire-share-url") share_url_value = share_url_input.get_attribute("value") - assert "acheteur_id=a1" in share_url_value, ( + assert "acheteur_id=123" in share_url_value, ( f"Share URL should contain acheteur_id param, got: {share_url_value}" ) @@ -211,6 +211,53 @@ def test_008_search_to_observatoire(dash_duo: DashComposite): time.sleep(1) # Allow callback chain to complete acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") - assert acheteur_input.get_attribute("value") == "a1", ( + assert acheteur_input.get_attribute("value") == "123", ( "acheteur_id input should be populated after navigating from search" ) + + +def test_009_observatoire_filter_persistence(dash_duo: DashComposite): + import time + + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Clear localStorage to start from a clean state + dash_duo.driver.execute_script("localStorage.clear()") + + # Navigate to observatoire without URL params + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + # Set the acheteur_id text input; press Enter to trigger the debounced save callback + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + dash_duo.clear_input(acheteur_input) + acheteur_input.send_keys("123") + acheteur_input.send_keys(Keys.ENTER) + + time.sleep(0.3) # allow the save callback to write to localStorage + + # Navigate away + dash_duo.wait_for_page(f"{dash_duo.server_url}/") + + # Navigate back without URL params + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + time.sleep(0.5) # allow restore callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "123", ( + "acheteur_id should be restored from localStorage after navigating back" + ) + + # Also verify URL params still override localStorage + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=123") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + time.sleep(0.5) + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "123", ( + "URL param acheteur_id should override the value stored in localStorage" + ) From 1fdcb12dd25918008b520df45a5bb172833f2805 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:32:43 +0100 Subject: [PATCH 25/75] feat: add dcc.Store and debounce text inputs on observatoire page #65 --- src/pages/observatoire.py | 50 +++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 6887f93..ec48c38 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -27,6 +27,8 @@ from src.figures import ( from src.utils import ( departements, df, + df_acheteurs, + df_titulaires, format_number, get_enum_values_as_dict, meta_content, @@ -118,8 +120,8 @@ def _apply_filters( layout = [ - dcc.Store(id="dashboard-filters"), dcc.Location(id="dashboard_url", refresh="callback-nav"), + dcc.Store(id="observatoire-filters", storage_type="local"), dbc.Modal( [ dbc.ModalHeader(dbc.ModalTitle("Montants")), @@ -146,7 +148,7 @@ Alors, on fait comment ? html.Div( className="container-fluid", children=[ - html.H2(name), + html.H2(children=[name], id="page_title"), dcc.Loading( overlay_style={"visibility": "visible", "filter": "blur(2px)"}, id="loading-statistques", @@ -175,6 +177,7 @@ Alors, on fait comment ? dcc.Input( id="dashboard_acheteur_id", placeholder="SIRET", + debounce=True, style={"width": "100%"}, ), ), @@ -207,6 +210,7 @@ Alors, on fait comment ? dcc.Input( id="dashboard_titulaire_id", placeholder="SIRET", + debounce=True, style={"width": "100%"}, ), ), @@ -473,7 +477,11 @@ def udpate_dashboard_cards( ) cards.append( make_card( - title="Catégorie d'acheteur", fig=donut_acheteur_categorie, lg=12, xl=8 + title="Catégorie d'acheteur", + subtitle="en nombre de marchés attribués", + fig=donut_acheteur_categorie, + lg=12, + xl=8, ) ) @@ -481,11 +489,21 @@ def udpate_dashboard_cards( lff, "titulaire_categorie", per_uid=False, nulls="?" ) cards.append( - make_card(title="Catégorie d'entreprise", fig=donut_titulaire_categorie) + make_card( + title="Catégorie d'entreprise", + subtitle="en nombre de marchés attribués", + fig=donut_titulaire_categorie, + ) ) donut_marche_type = make_donut(lff, "type", per_uid=True, nulls="?") - cards.append(make_card(title="Type d'achat", fig=donut_marche_type)) + cards.append( + make_card( + title="Type d'achat", + subtitle="en nombre de marchés attribués", + fig=donut_marche_type, + ) + ) geographic_maps: list[dbc.Col] = get_geographic_maps(dff) @@ -573,3 +591,25 @@ def download_observatoire( ) def toggle_montant_modal(n_triggers, _close): return isinstance(ctx.triggered_id, dict) and any(n_triggers) + + +@callback( + Output("page_title", "children"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_titulaire_id", "value"), + prevent_initial_call=False, +) +def add_organization_name_in_title(acheteur_id, titulaire_id): + def lookup_nom(df_org, id_col, nom_col, org_id): + match = df_org.filter(pl.col(id_col) == org_id) + return match[nom_col].item(0) if match.height >= 1 else None + + if acheteur_id and len(acheteur_id) == 12: + if nom := lookup_nom(df_acheteurs, "acheteur_id", "acheteur_nom", acheteur_id): + return f"{name} - {nom}" + elif titulaire_id and len(titulaire_id) == 12: + if nom := lookup_nom( + df_titulaires, "titulaire_id", "titulaire_nom", titulaire_id + ): + return f"{name} - {nom}" + return name From 051e908bd18f983d3f3a70951dc5d4f17c7b3b8c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:33:15 +0100 Subject: [PATCH 26/75] feat: save observatoire filters to localStorage on change #65 --- src/pages/observatoire.py | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index ec48c38..53dfddf 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -326,6 +326,46 @@ def restore_filters_from_url(search): return acheteur_id, titulaire_id +@callback( + Output("observatoire-filters", "data"), + Input("dashboard_year", "value"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_acheteur_categorie", "value"), + Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_titulaire_id", "value"), + Input("dashboard_titulaire_categorie", "value"), + Input("dashboard_titulaire_departement_code", "value"), + Input("dashboard_marche_type", "value"), + Input("dashboard_marche_considerationsSociales", "value"), + Input("dashboard_marche_considerationsEnvironnementales", "value"), + prevent_initial_call=True, +) +def save_filters_to_storage( + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, +): + return { + "year": year, + "acheteur_id": acheteur_id, + "acheteur_categorie": acheteur_categorie, + "acheteur_departement_code": acheteur_departement_code, + "titulaire_id": titulaire_id, + "titulaire_categorie": titulaire_categorie, + "titulaire_departement_code": titulaire_departement_code, + "marche_type": marche_type, + "considerations_sociales": considerations_sociales, + "considerations_environnementales": considerations_environnementales, + } + + @callback( Output("observatoire-share-url", "value"), Output("observatoire-copy-container", "children"), From b4a42449ad5a832fa6c2ea172b30f419255db3aa Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:35:31 +0100 Subject: [PATCH 27/75] feat: restore observatoire filters from localStorage on page load #65 --- src/pages/observatoire.py | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 53dfddf..aa13f18 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -303,27 +303,54 @@ Alors, on fait comment ? @callback( + Output("dashboard_year", "value"), Output("dashboard_acheteur_id", "value"), + Output("dashboard_acheteur_categorie", "value"), + Output("dashboard_acheteur_departement_code", "value"), Output("dashboard_titulaire_id", "value"), - # Output("dashboard_url", "search"), + Output("dashboard_titulaire_categorie", "value"), + Output("dashboard_titulaire_departement_code", "value"), + Output("dashboard_marche_type", "value"), + Output("dashboard_marche_considerationsSociales", "value"), + Output("dashboard_marche_considerationsEnvironnementales", "value"), Input("dashboard_url", "search"), + Input("dashboard_url", "pathname"), + State("observatoire-filters", "data"), ) -def restore_filters_from_url(search): - if not search: - return no_update, no_update +def restore_filters(search, _pathname, stored_filters): + if search: + params = urllib.parse.parse_qs(search.lstrip("?")) + acheteur_id = (params.get("acheteur_id") or [None])[0] or None + titulaire_id = (params.get("titulaire_id") or [None])[0] or None + if acheteur_id or titulaire_id: + return ( + None, + acheteur_id, + None, + None, + titulaire_id, + None, + None, + None, + None, + None, + ) - params = urllib.parse.parse_qs(search.lstrip("?")) + if stored_filters: + return ( + stored_filters.get("year"), + stored_filters.get("acheteur_id"), + stored_filters.get("acheteur_categorie"), + stored_filters.get("acheteur_departement_code"), + stored_filters.get("titulaire_id"), + stored_filters.get("titulaire_categorie"), + stored_filters.get("titulaire_departement_code"), + stored_filters.get("marche_type"), + stored_filters.get("considerations_sociales"), + stored_filters.get("considerations_environnementales"), + ) - def get_param_value(key): - values = params.get(key) - if values and values[0] is not None: - return values[0] - return no_update - - acheteur_id = get_param_value("acheteur_id") - titulaire_id = get_param_value("titulaire_id") - - return acheteur_id, titulaire_id + return (no_update,) * 10 @callback( From f8fffb6fa444afeef104314036c95e6c78841362 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:40:59 +0100 Subject: [PATCH 28/75] get_top_org un peu plus configurable #65 --- src/callbacks.py | 8 ++++---- src/pages/acheteur.py | 2 +- src/pages/titulaire.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/callbacks.py b/src/callbacks.py index 0ca434e..9d791db 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -5,14 +5,14 @@ from src.figures import DataTable from utils import add_links_in_dict, format_values, setup_table_columns -def get_top_org_table(data, org_type: str): +def get_top_org_table(data, org_type: str, extra_columns: list): dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) if dff.height == 0: return html.Div() - dff = dff.select( - ["uid", f"{org_type}_id", f"{org_type}_nom", "titulaire_distance", "montant"] - ) + extra_columns = [] if extra_columns is None else extra_columns + + dff = dff.select(["uid", f"{org_type}_id", f"{org_type}_nom"] + extra_columns) dff_nb = dff.group_by( f"{org_type}_id", f"{org_type}_nom", "titulaire_distance" ).agg(pl.len().alias("Attributions"), pl.sum("montant").alias("montant")) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 199c757..76cd2d6 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -340,7 +340,7 @@ def get_last_marches_data( Input(component_id="acheteur_data", component_property="data"), ) def get_top_titulaires(data): - return get_top_org_table(data, "titulaire") + return get_top_org_table(data, "titulaire", ["titulaire_distance", "montant"]) @callback( diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index b7e9001..ecee685 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -354,7 +354,7 @@ def get_last_marches_data( Input(component_id="titulaire_data", component_property="data"), ) def get_top_acheteurs(data): - return get_top_org_table(data, "acheteur") + return get_top_org_table(data, "acheteur", ["titulaire_distance", "montant"]) @callback( From 26dd2aaf1a1218a90d749809f64438289dfe7e5e Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:41:46 +0100 Subject: [PATCH 29/75] Correction de l'injection du nom d'org dans le titre #65 --- src/figures.py | 5 ----- src/pages/observatoire.py | 4 ++-- tests/conftest.py | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/figures.py b/src/figures.py index 8337f9c..8e8afbf 100644 --- a/src/figures.py +++ b/src/figures.py @@ -89,11 +89,6 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): .sort(by=[type_date, "len"], descending=True) ) - # lff = lff.with_columns( - # pl.when(pl.col("sourceDataset").is_null()).then( - # pl.lit("Source inconnue")).alias("sourceDataset") - # ) - lff = lff.sort(by=["sourceDataset"], descending=False) dff: pl.DataFrame = lff.collect(engine="streaming") diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index aa13f18..404e14c 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -671,10 +671,10 @@ def add_organization_name_in_title(acheteur_id, titulaire_id): match = df_org.filter(pl.col(id_col) == org_id) return match[nom_col].item(0) if match.height >= 1 else None - if acheteur_id and len(acheteur_id) == 12: + if acheteur_id and len(acheteur_id) == 14: if nom := lookup_nom(df_acheteurs, "acheteur_id", "acheteur_nom", acheteur_id): return f"{name} - {nom}" - elif titulaire_id and len(titulaire_id) == 12: + elif titulaire_id and len(titulaire_id) == 14: if nom := lookup_nom( df_titulaires, "titulaire_id", "titulaire_nom", titulaire_id ): diff --git a/tests/conftest.py b/tests/conftest.py index f0f2b00..78f1362 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,9 +13,9 @@ def test_data(): "uid": "1", "id": "1", "acheteur_nom": "ACHETEUR 1", - "acheteur_id": "a1", + "acheteur_id": "123", "titulaire_nom": "TITULAIRE 1", - "titulaire_id": "t1", + "titulaire_id": "345", "montant": 10, "dateNotification": datetime.date(2025, 1, 1), "codeCPV": "71600000", From f2046d6ba70f60831e2bee9291badeb855e6a784 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:46:38 +0100 Subject: [PATCH 30/75] =?UTF-8?q?Meilleure=20int=C3=A9gration=20du=20nom?= =?UTF-8?q?=20de=20l'org=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 404e14c..87c3470 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -673,10 +673,16 @@ def add_organization_name_in_title(acheteur_id, titulaire_id): if acheteur_id and len(acheteur_id) == 14: if nom := lookup_nom(df_acheteurs, "acheteur_id", "acheteur_nom", acheteur_id): - return f"{name} - {nom}" + return [ + name, + html.Small(nom, className="text-muted d-block fw-normal fs-5"), + ] elif titulaire_id and len(titulaire_id) == 14: if nom := lookup_nom( df_titulaires, "titulaire_id", "titulaire_nom", titulaire_id ): - return f"{name} - {nom}" + return [ + name, + html.Small(nom, className="text-muted d-block fw-normal fs-5"), + ] return name From c77511d4e85fe63c48b2bdc00097828265c07f93 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:57:30 +0100 Subject: [PATCH 31/75] Taille de donut flexible #65 --- src/figures.py | 4 ++++ src/pages/observatoire.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/figures.py b/src/figures.py index 8e8afbf..451f24c 100644 --- a/src/figures.py +++ b/src/figures.py @@ -609,6 +609,7 @@ def make_donut( names_col, per_uid: bool, nulls="?", + potentially_many_names: bool = False, ): title = data_schema[names_col]["title"] lff = lff.rename({names_col: title}) @@ -620,6 +621,7 @@ def make_donut( lff = lff.group_by(title).len("Nombre") lff = lff.with_columns(pl.col(title).replace(None, pl.lit(nulls))) dff = lff.collect(engine="streaming") + nb_names = dff[title].n_unique() dff = dff.with_columns( pl.col("Nombre") .map_elements(format_number, return_dtype=pl.String) @@ -639,6 +641,8 @@ def make_donut( ) fig = fig.update_layout(showlegend=False, font=dict(size=14)) graph = dcc.Graph(figure=fig) + if potentially_many_names: + return graph, nb_names return graph diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 87c3470..6ca7d10 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -539,16 +539,20 @@ def udpate_dashboard_cards( cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) - donut_acheteur_categorie = make_donut( - lff, "acheteur_categorie", nulls="Autres", per_uid=True + donut_acheteur_categorie, nb_acheteur_categories = make_donut( + lff, + "acheteur_categorie", + nulls="Autres", + per_uid=True, + potentially_many_names=True, ) cards.append( make_card( title="Catégorie d'acheteur", subtitle="en nombre de marchés attribués", fig=donut_acheteur_categorie, - lg=12, - xl=8, + lg=12 if nb_acheteur_categories > 4 else 6, + xl=8 if nb_acheteur_categories > 4 else 4, ) ) From 78d528f75bf7b3f7f46a08f12d36d513960d54e6 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 21:13:21 +0100 Subject: [PATCH 32/75] Filtre par montant #65 --- src/assets/css/style.css | 13 ++++++++- src/pages/observatoire.py | 60 ++++++++++++++++++++++++++++++++++++++- tests/test_main.py | 41 ++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 12dd156..f93c94a 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -201,7 +201,8 @@ p.version > a { margin-bottom: 6px; } -#filters input[type="text"] { +#filters input[type="text"], +#filters input[type="number"] { border: 1px #ccc solid; border-radius: 3px; padding-left: 8px; @@ -578,3 +579,13 @@ summary > h4 { display: none; } } + +input[type="number"]::-webkit-outer-spin-button, +input[type="number"]::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; +} + +input[type="number"] { + -moz-appearance: textfield; +} diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 6ca7d10..f2bbe65 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -67,6 +67,8 @@ def _apply_filters( marche_type, considerations_sociales, considerations_environnementales, + montant_min=None, + montant_max=None, ) -> pl.LazyFrame: if year: lff = lff.filter(pl.col("dateNotification").dt.year() == int(year)) @@ -116,6 +118,12 @@ def _apply_filters( > 0 ) + if montant_min is not None: + lff = lff.filter(pl.col("montant") >= montant_min) + + if montant_max is not None: + lff = lff.filter(pl.col("montant") <= montant_max) + return lff @@ -271,6 +279,32 @@ Alors, on fait comment ? ), ), ), + dbc.Row( + [ + dbc.Col( + dcc.Input( + id="dashboard_montant_min", + placeholder="Montant min.", + type="number", + min=0, + debounce=True, + style={"width": "100%"}, + ), + width=6, + ), + dbc.Col( + dcc.Input( + id="dashboard_montant_max", + placeholder="Montant max.", + type="number", + min=0, + debounce=True, + style={"width": "100%"}, + ), + width=6, + ), + ] + ), dcc.Download(id="download-observatoire"), dbc.Button( "Télécharger au format Excel", @@ -313,6 +347,8 @@ Alors, on fait comment ? Output("dashboard_marche_type", "value"), Output("dashboard_marche_considerationsSociales", "value"), Output("dashboard_marche_considerationsEnvironnementales", "value"), + Output("dashboard_montant_min", "value"), + Output("dashboard_montant_max", "value"), Input("dashboard_url", "search"), Input("dashboard_url", "pathname"), State("observatoire-filters", "data"), @@ -334,6 +370,8 @@ def restore_filters(search, _pathname, stored_filters): None, None, None, + None, + None, ) if stored_filters: @@ -348,9 +386,11 @@ def restore_filters(search, _pathname, stored_filters): stored_filters.get("marche_type"), stored_filters.get("considerations_sociales"), stored_filters.get("considerations_environnementales"), + stored_filters.get("montant_min"), + stored_filters.get("montant_max"), ) - return (no_update,) * 10 + return (no_update,) * 12 @callback( @@ -365,6 +405,8 @@ def restore_filters(search, _pathname, stored_filters): Input("dashboard_marche_type", "value"), Input("dashboard_marche_considerationsSociales", "value"), Input("dashboard_marche_considerationsEnvironnementales", "value"), + Input("dashboard_montant_min", "value"), + Input("dashboard_montant_max", "value"), prevent_initial_call=True, ) def save_filters_to_storage( @@ -378,6 +420,8 @@ def save_filters_to_storage( marche_type, considerations_sociales, considerations_environnementales, + montant_min, + montant_max, ): return { "year": year, @@ -390,6 +434,8 @@ def save_filters_to_storage( "marche_type": marche_type, "considerations_sociales": considerations_sociales, "considerations_environnementales": considerations_environnementales, + "montant_min": montant_min, + "montant_max": montant_max, } @@ -453,6 +499,8 @@ def sync_observatoire_share_url(acheteur_id, titulaire_id, href): Input("dashboard_marche_type", "value"), Input("dashboard_marche_considerationsSociales", "value"), Input("dashboard_marche_considerationsEnvironnementales", "value"), + Input("dashboard_montant_min", "value"), + Input("dashboard_montant_max", "value"), ) def udpate_dashboard_cards( dashboard_year, @@ -465,6 +513,8 @@ def udpate_dashboard_cards( dashboard_marche_type, dashboard_marche_considerations_sociales, dashboard_marche_considerations_environnementales, + dashboard_montant_min, + dashboard_montant_max, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -490,6 +540,8 @@ def udpate_dashboard_cards( dashboard_marche_type, dashboard_marche_considerations_sociales, dashboard_marche_considerations_environnementales, + montant_min=dashboard_montant_min, + montant_max=dashboard_montant_max, ) # Génération des métriques @@ -618,6 +670,8 @@ def udpate_dashboard_cards( State("dashboard_marche_type", "value"), State("dashboard_marche_considerationsSociales", "value"), State("dashboard_marche_considerationsEnvironnementales", "value"), + State("dashboard_montant_min", "value"), + State("dashboard_montant_max", "value"), prevent_initial_call=True, ) def download_observatoire( @@ -632,6 +686,8 @@ def download_observatoire( marche_type, considerations_sociales, considerations_environnementales, + montant_min, + montant_max, ): lff = _apply_filters( df.lazy(), @@ -645,6 +701,8 @@ def download_observatoire( marche_type, considerations_sociales, considerations_environnementales, + montant_min=montant_min, + montant_max=montant_max, ) def to_bytes(buffer): diff --git a/tests/test_main.py b/tests/test_main.py index f9d36b7..6a13762 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -216,6 +216,47 @@ def test_008_search_to_observatoire(dash_duo: DashComposite): ) +def test_010_observatoire_montant_filter(): + import datetime + + import polars as pl + + from pages.observatoire import _apply_filters + from src.app import ( + app, # noqa: F401 – instantiates the Dash app before register_page() calls + ) + + data = pl.DataFrame( + { + "uid": ["1", "2", "3"], + "montant": [100.0, 500.0, 1000.0], + "dateNotification": [datetime.date(2025, 1, 1)] * 3, + } + ) + + def apply(min_val=None, max_val=None): + return _apply_filters( + data.lazy(), + year="2025", + acheteur_id=None, + acheteur_categorie=None, + acheteur_departement_code=None, + titulaire_id=None, + titulaire_categorie=None, + titulaire_departement_code=None, + marche_type=None, + considerations_sociales=None, + considerations_environnementales=None, + montant_min=min_val, + montant_max=max_val, + ).collect() + + assert apply().height == 3 + assert apply(min_val=400).height == 2 # 500, 1000 + assert apply(max_val=500).height == 2 # 100, 500 + assert apply(min_val=200, max_val=600).height == 1 # 500 only + + def test_009_observatoire_filter_persistence(dash_duo: DashComposite): import time From 125c520c19da26bf94e047660b75b4d193761f14 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 21:47:29 +0100 Subject: [PATCH 33/75] Claude memory --- CLAUDE.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b4ead7d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,85 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +**decp.info** is a French public procurement data explorer — a Dash (Python) web app for browsing, filtering, and visualizing _Données Essentielles de la Commande Publique_ (DECP). The UI is in French. + +## Commands + +### Setup + +```bash +python -m venv .venv && source .venv/bin/activate +pip install ".[dev]" +cp template.env .env # then customize .env +``` + +### Development + +```bash +uv run run.py # starts Dash with debug=True and hot reload +``` + +### Production + +```bash +gunicorn app:server +``` + +### Tests + +```bash +uv run pytest # run all tests (Selenium-based integration tests) +uv run pytest tests/test_main.py::test_001_logo_and_search # run a single test +``` + +Tests require a running Chrome/Chromium browser. They use `DashComposite` from `dash[testing]` with Selenium WebDriver. + +## Architecture + +### Multi-page Dash app + +- `src/app.py` — creates the Dash app instance, navbar, SEO endpoints (robots.txt, sitemap.xml), Matomo analytics +- `src/pages/*.py` — each page registers itself with `@register_page()` and owns its own layout and callbacks +- `run.py` — dev entry point; exports `server` (Flask) for gunicorn + +### Key pages + +| Page | URL | Purpose | +| ----------------- | --------------- | -------------------------------------- | +| `recherche.py` | `/` | Search homepage for buyers/contractors | +| `acheteur.py` | `/acheteur` | Buyer detail with stats, charts, maps | +| `titulaire.py` | `/titulaire` | Contractor detail | +| `tableau.py` | `/tableau` | Filterable data table with exports | +| `marche.py` | `/marche` | Individual contract detail | +| `observatoire.py` | `/observatoire` | An interactive analytics dashboard | + +### Data layer + +- Data is stored as **Parquet** and loaded with **Polars** (fast columnar operations) +- Path set via `DATA_FILE_PARQUET_PATH` env var; tests use `tests/test.parquet` +- `src/utils.py` — filtering helpers, search (`search_org`), link generation, geographic data loading +- `src/callbacks.py` — shared Dash callbacks (e.g. `get_top_org_table`) +- `src/figures.py` — chart and map components (Plotly Express, Dash Leaflet with marker clustering) +- a Parquet file with production data is located at `../decp-processing/decp_prod.parquet` (~ 1,5 million records) +- the TableSchema of the dataset with the list of field and their definition is located at `../decp-processing/reference/base_schema.json` +- `tests/test.parquet` is very small and may not contain all possible columns, only those necessary for testing + +### UI stack + +- **Dash 3.4** + **Dash Bootstrap Components** for layout +- **Plotly Express** for charts +- **Dash Leaflet** + **Dash Extensions** for interactive maps with clustering +- Custom CSS in `src/assets/css/` + +### Environment + +- `DEVELOPMENT=true` enables debug logging and is set automatically during tests +- `.env` file is required at runtime (copy from `template.env`) + +### Deployment + +- `main` branch → manual deploy to decp.info via GitHub Actions +- `dev` branch → auto-deploy to test.decp.info via GitHub Actions From 9da3d9ce34ed588218252b5dd0d6b4dfa3be03a5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 21:48:23 +0100 Subject: [PATCH 34/75] =?UTF-8?q?Ne=20pas=20supprimer=20les=20donn=C3=A9es?= =?UTF-8?q?=20de=20test=20=C3=A0=20la=20fin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 78f1362..1874b52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,10 +48,6 @@ def test_data(): pl.DataFrame(data).write_parquet("tests/test.parquet") yield path - if os.path.exists(path): - os.unlink(path) - print(path, "deleted") - def pytest_setup_options(): options = Options() From 1ffa995a4a1c0ffe77fccfdc29c04777a946bd85 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:02:20 +0100 Subject: [PATCH 35/75] docs: spec for distance histogram on observatoire, acheteur, titulaire pages Co-Authored-By: Claude Sonnet 4.6 --- .../2026-03-18-distance-histogram-design.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-18-distance-histogram-design.md diff --git a/docs/superpowers/specs/2026-03-18-distance-histogram-design.md b/docs/superpowers/specs/2026-03-18-distance-histogram-design.md new file mode 100644 index 0000000..091e274 --- /dev/null +++ b/docs/superpowers/specs/2026-03-18-distance-histogram-design.md @@ -0,0 +1,64 @@ +# Distance Histogram — Design Spec + +**Date:** 2026-03-18 +**Branch:** feature/65_observatoire + +## Goal + +Display the distribution of distances (in km) between buyers and winning contractors, to help users assess whether a buyer or contractor tends to deal locally or at a national scale. + +## Data + +- Column: `titulaire_distance` (`Int64`, km) +- Measured at address level — values are always > 0, no zero-handling needed +- Already selected in the observatoire LazyFrame via `cs.starts_with("titulaire")` +- Already available on acheteur and titulaire detail pages + +## Figure Function + +**Location:** `src/figures.py` + +**Signature:** + +```python +def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: +``` + +**Behaviour:** + +- Collects `titulaire_distance` from the LazyFrame, drops nulls +- If the resulting DataFrame is empty after dropping nulls, `px.histogram` produces a blank figure without errors — no guard logic needed. The order of operations must be: drop nulls → log-transform → histogram +- Drop nulls first, then pre-log-transform the column (`pl.col("titulaire_distance").log(10)`) so bins are truly equal-width on a log scale. Use `px.histogram` with `nbins=50` on the transformed values +- Set custom X-axis tick values at powers of 10 (1, 10, 100, 1000, 10000) with km labels, using `fig.update_xaxes(tickvals=[0,1,2,3,4], ticktext=["1","10","100","1 000","10 000"])` +- Y axis: count of contracts +- French axis labels: x = `"Distance (km)"`, y = `"Nombre de marchés"` +- Returns a `dcc.Graph` + +## Integration + +### Observatoire (`src/pages/observatoire.py`) + +- `get_distance_histogram` imported and called inside `udpate_dashboard_cards` +- Result wrapped in `make_card(title="Distance acheteur–titulaire", subtitle="en nombre de marchés, échelle logarithmique", fig=...)` +- Card appended to the `cards` list alongside existing donuts and charts +- No changes to the data pipeline — `titulaire_distance` is already in the LazyFrame + +### Acheteur page (`src/pages/acheteur.py`) + +The acheteur page uses a `dcc.Store` (`acheteur_data`) that holds serialised contract rows as a list of dicts. The integration follows the existing pattern used by other chart callbacks on this page: + +- Add a new `html.Div(id="acheteur-distance-histogram")` placeholder in the layout +- Add a new callback with `Input("acheteur_data", "data")` that: + - Reconstructs `pl.LazyFrame(data)` from the store + - Calls `get_distance_histogram(lff)` + - Wraps the result in `make_card(...)` and returns it to the placeholder div + +### Titulaire page (`src/pages/titulaire.py`) + +Same pattern as acheteur: `dcc.Store` (`titulaire_data`) → new callback → `html.Div` placeholder. + +## Out of Scope + +- Filtering by distance range (could be a future filter on the observatoire page) +- Showing distance on a map or as a trend over time +- Bucket-based (named zone) grouping From 24db748b6a0a8184fd127c50ba63df729c955fdb Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:15:26 +0100 Subject: [PATCH 36/75] feat: add get_distance_histogram figure function Implement get_distance_histogram that creates a histogram of titulaire distances with logarithmic scale. Add 3 unit tests covering basic functionality, null handling, and edge cases. Also add DATA_SCHEMA_PATH to pytest env config. Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 3 ++- src/figures.py | 18 ++++++++++++++++++ tests/test_main.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2befbcf..b4c27e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ testpaths = [ ] env = [ "DATA_FILE_PARQUET_PATH=tests/test.parquet", - "DEVELOPMENT=true" + "DEVELOPMENT=true", + "DATA_SCHEMA_PATH=/home/colin/git/decp-processing/dist/schema.json" ] addopts = "-p no:warnings" diff --git a/src/figures.py b/src/figures.py index 451f24c..ea87874 100644 --- a/src/figures.py +++ b/src/figures.py @@ -579,6 +579,24 @@ def make_clusters_map(region: dict) -> dl.Map: return leaflet_map +def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: + dff = lff.select("titulaire_distance").drop_nulls().collect() + dff = dff.with_columns(pl.col("titulaire_distance").log(10)) + fig = px.histogram( + dff, + x="titulaire_distance", + nbins=50, + labels={"titulaire_distance": "Distance (km)"}, + ) + fig.update_xaxes( + tickvals=[0, 1, 2, 3, 4], + ticktext=["1", "10", "100", "1 000", "10 000"], + title_text="Distance (km)", + ) + fig.update_yaxes(title_text="Nombre de marchés") + return dcc.Graph(figure=fig) + + def make_card( title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4 ) -> dbc.Col: diff --git a/tests/test_main.py b/tests/test_main.py index 6a13762..a50d807 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -302,3 +302,36 @@ def test_009_observatoire_filter_persistence(dash_duo: DashComposite): assert acheteur_input.get_attribute("value") == "123", ( "URL param acheteur_id should override the value stored in localStorage" ) + + +def test_get_distance_histogram_returns_graph(): + import polars as pl + from dash import dcc + + from src.figures import get_distance_histogram + + lff = pl.LazyFrame({"titulaire_distance": [1, 10, 100, 500, 1000]}) + result = get_distance_histogram(lff) + assert isinstance(result, dcc.Graph) + + +def test_get_distance_histogram_handles_nulls(): + import polars as pl + from dash import dcc + + from src.figures import get_distance_histogram + + lff = pl.LazyFrame({"titulaire_distance": [None, None, 50]}) + result = get_distance_histogram(lff) + assert isinstance(result, dcc.Graph) + + +def test_get_distance_histogram_all_nulls(): + import polars as pl + from dash import dcc + + from src.figures import get_distance_histogram + + lff = pl.LazyFrame({"titulaire_distance": pl.Series([], dtype=pl.Int64)}) + result = get_distance_histogram(lff) + assert isinstance(result, dcc.Graph) From f2af09bb3581214f86070aaf15f26d28049cefbe Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:22:54 +0100 Subject: [PATCH 37/75] fix: use streaming collect and filter zero distances in get_distance_histogram --- src/figures.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/figures.py b/src/figures.py index ea87874..ea8d905 100644 --- a/src/figures.py +++ b/src/figures.py @@ -580,7 +580,12 @@ def make_clusters_map(region: dict) -> dl.Map: def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: - dff = lff.select("titulaire_distance").drop_nulls().collect() + dff = ( + lff.select("titulaire_distance") + .drop_nulls() + .filter(pl.col("titulaire_distance") > 0) + .collect(engine="streaming") + ) dff = dff.with_columns(pl.col("titulaire_distance").log(10)) fig = px.histogram( dff, From 44d2d7d2c1fa9fb2b5b7e83169a404a283ac9352 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:24:01 +0100 Subject: [PATCH 38/75] feat: add distance histogram card to observatoire dashboard Integrate the get_distance_histogram function into the observatoire dashboard to display buyer-contractor distance distribution on a logarithmic scale. Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index f2bbe65..52ccbd4 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -19,6 +19,7 @@ from dash import ( from src.figures import ( get_barchart_sources, + get_distance_histogram, get_duplicate_matrix, get_geographic_maps, make_card, @@ -628,6 +629,15 @@ def udpate_dashboard_cards( ) ) + distance_histogram = get_distance_histogram(lff) + cards.append( + make_card( + title="Distance acheteur–titulaire", + subtitle="en nombre de marchés, échelle logarithmique", + fig=distance_histogram, + ) + ) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) other_cards = [] From b08d517f3667a764da345f6f737384ca8968c8ee Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:26:57 +0100 Subject: [PATCH 39/75] feat: add distance histogram to acheteur detail page --- src/pages/acheteur.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 76cd2d6..c6ffd9d 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -16,7 +16,13 @@ from dash import ( ) from src.callbacks import get_top_org_table -from src.figures import DataTable, make_column_picker, point_on_map +from src.figures import ( + DataTable, + get_distance_histogram, + make_card, + make_column_picker, + point_on_map, +) from src.utils import ( columns, df, @@ -140,6 +146,7 @@ layout = [ html.Div(className="marches_table", id="top10_titulaires"), ], ), + html.Div(id="acheteur-distance-histogram"), ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -476,3 +483,17 @@ def toggle_acheteur_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] + + +@callback( + Output("acheteur-distance-histogram", "children"), + Input("acheteur_data", "data"), +) +def update_acheteur_distance_histogram(data): + lff = pl.LazyFrame(data) + fig = get_distance_histogram(lff) + return make_card( + title="Distance acheteur–titulaire", + subtitle="en nombre de marchés, échelle logarithmique", + fig=fig, + ) From 858ab6c61ab1d65395e44e667127cba83ae5cb46 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:28:34 +0100 Subject: [PATCH 40/75] feat: add distance histogram to titulaire detail page --- src/pages/titulaire.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index ecee685..52ba25e 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -16,7 +16,13 @@ from dash import ( ) from src.callbacks import get_top_org_table -from src.figures import DataTable, make_column_picker, point_on_map +from src.figures import ( + DataTable, + get_distance_histogram, + make_card, + make_column_picker, + point_on_map, +) from src.utils import ( columns, df, @@ -140,6 +146,7 @@ layout = [ html.Div(className="marches_table", id="top10_acheteurs"), ], ), + html.Div(id="titulaire-distance-histogram"), ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -490,3 +497,21 @@ def toggle_titulaire_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] + + +@callback( + Output("titulaire-distance-histogram", "children"), + Input("titulaire_data", "data"), +) +def update_titulaire_distance_histogram(data): + lff = pl.LazyFrame(data) + if "titulaire_distance" in lff.collect_schema().names(): + lff = lff.with_columns( + pl.col("titulaire_distance").cast(pl.Float64, strict=False) + ) + fig = get_distance_histogram(lff) + return make_card( + title="Distance acheteur–titulaire", + subtitle="en nombre de marchés, échelle logarithmique", + fig=fig, + ) From 32aa8777971384d1dfe99f8cc7b37b5fbfd8982f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:40:28 +0100 Subject: [PATCH 41/75] fix: guard against missing titulaire_distance column in get_distance_histogram --- src/figures.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/figures.py b/src/figures.py index ea8d905..14b624a 100644 --- a/src/figures.py +++ b/src/figures.py @@ -580,12 +580,15 @@ def make_clusters_map(region: dict) -> dl.Map: def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: - dff = ( - lff.select("titulaire_distance") - .drop_nulls() - .filter(pl.col("titulaire_distance") > 0) - .collect(engine="streaming") - ) + if "titulaire_distance" not in lff.collect_schema().names(): + dff = pl.DataFrame({"titulaire_distance": pl.Series([], dtype=pl.Float64)}) + else: + dff = ( + lff.select("titulaire_distance") + .drop_nulls() + .filter(pl.col("titulaire_distance") > 0) + .collect(engine="streaming") + ) dff = dff.with_columns(pl.col("titulaire_distance").log(10)) fig = px.histogram( dff, From 8a61d3268aae4f4637e59b22e62336dea0716b69 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:53:17 +0100 Subject: [PATCH 42/75] =?UTF-8?q?fix:=20lecture=20du=20dataframe=20plus=20?= =?UTF-8?q?flexible=20dans=20la=20cr=C3=A9ation=20de=20l'histogram?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index c6ffd9d..a51846a 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -490,7 +490,7 @@ def reset_view(n_clicks): Input("acheteur_data", "data"), ) def update_acheteur_distance_histogram(data): - lff = pl.LazyFrame(data) + lff = pl.LazyFrame(data, strict=False, infer_schema_length=5000) fig = get_distance_histogram(lff) return make_card( title="Distance acheteur–titulaire", From 6a6be51455882c089e16b5afc11ec495469087c4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 23:21:17 +0100 Subject: [PATCH 43/75] refactor: replace CSS grid layout with Dash Bootstrap Components grid Replace the custom CSS grid (`.wrapper`, `.org_*`, `.results_*` classes) in acheteur, titulaire, and recherche pages with dbc.Row/dbc.Col. Co-Authored-By: Claude Sonnet 4.6 --- src/assets/css/style.css | 45 -------------- src/pages/acheteur.py | 128 +++++++++++++++++++++++---------------- src/pages/recherche.py | 18 +++--- src/pages/titulaire.py | 128 +++++++++++++++++++++++---------------- 4 files changed, 163 insertions(+), 156 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index f93c94a..d76b6bb 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -94,12 +94,6 @@ button:hover:not([disabled]) { padding: 28px 24px 0 24px; } -.wrapper { - display: grid; - grid-gap: 10px; - margin-bottom: 50px; - justify-content: space-between; -} #header > * { margin: 0 0 20px 0px; @@ -180,15 +174,6 @@ p.version > a { margin-right: 12px; } -.results_acheteur { - grid-column: 1; - grid-row: 1; -} - -.results_titulaire { - grid-column: 2; - grid-row: 1; -} /* --- Dashboard inputs --- */ @@ -446,40 +431,10 @@ input[type="checkbox"] { margin-bottom: 16px; } -.org_title { - grid-column: 1 / 3; - grid-row: 1; -} - -.org_year { - grid-column: 3; - grid-row: 1; -} - -.org_infos { - grid-column: 1; - grid-row: 2; -} - .org_infos > p { margin: 8px 0; } -.org_stats { - grid-column: 2; - grid-row: 2; -} - -.org_map { - grid-column: 3; - grid-row: 2; -} - -.org_top { - grid-column: 1/3; - grid-row: 3; -} - /* --- About Page (A Propos) --- */ .a-propos-container { display: flex; diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index a51846a..9be863c 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -82,68 +82,94 @@ layout = [ html.Div( children=[ html.Div( - className="wrapper", + style={"marginBottom": "50px"}, children=[ - html.H2( - className="org_title", + dbc.Row( + className="mb-2", children=[ - html.Span(id="acheteur_siret"), - " - ", - html.Span(id="acheteur_nom"), - ], - ), - html.Div( - className="org_year", - children=dcc.Dropdown( - id="acheteur_year", - options=["Toutes les années"] - + [ - str(year) - for year in range( - 2018, int(datetime.date.today().year) + 1 - ) - ], - placeholder="Année", - ), - ), - html.Div( - className="org_infos", - children=[ - # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. - html.P(["Commune : ", html.Strong(id="acheteur_commune")]), - html.P( - [ - "Département : ", - html.Strong(id="acheteur_departement"), - ] + dbc.Col( + html.H2( + children=[ + html.Span(id="acheteur_siret"), + " - ", + html.Span(id="acheteur_nom"), + ], + ), + width=8, ), - html.P(["Région : ", html.Strong(id="acheteur_region")]), - html.A( - id="acheteur_lien_annuaire", - children="Plus de détails sur l'Annuaire des entreprises", + dbc.Col( + dcc.Dropdown( + id="acheteur_year", + options=["Toutes les années"] + + [ + str(year) + for year in range( + 2018, int(datetime.date.today().year) + 1 + ) + ], + placeholder="Année", + ), + width=4, ), ], ), - html.Div( - className="org_stats", + dbc.Row( + className="mb-2", children=[ - html.P(id="acheteur_titre_stats"), - html.P(id="acheteur_marches_attribues"), - html.P(id="acheteur_titulaires_differents"), - html.Button( - "Téléchargement au format Excel", - id="btn-download-data-acheteur", - className="btn btn-primary", + dbc.Col( + html.Div( + className="org_infos", + children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. + html.P(["Commune : ", html.Strong(id="acheteur_commune")]), + html.P( + [ + "Département : ", + html.Strong(id="acheteur_departement"), + ] + ), + html.P(["Région : ", html.Strong(id="acheteur_region")]), + html.A( + id="acheteur_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + ), + ], + ), + width=4, + ), + dbc.Col( + html.Div( + children=[ + html.P(id="acheteur_titre_stats"), + html.P(id="acheteur_marches_attribues"), + html.P(id="acheteur_titulaires_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-data-acheteur", + className="btn btn-primary", + ), + dcc.Download(id="download-data-acheteur"), + ], + ), + width=4, + ), + dbc.Col( + html.Div(id="acheteur_map"), + width=4, ), - dcc.Download(id="download-data-acheteur"), ], ), - html.Div(className="org_map", id="acheteur_map"), - html.Div( - className="org_top", + dbc.Row( children=[ - html.H3("Top titulaires"), - html.Div(className="marches_table", id="top10_titulaires"), + dbc.Col( + html.Div( + children=[ + html.H3("Top titulaires"), + html.Div(className="marches_table", id="top10_titulaires"), + ], + ), + width=8, + ), ], ), html.Div(id="acheteur-distance-histogram"), diff --git a/src/pages/recherche.py b/src/pages/recherche.py index 44b1271..b308cf8 100644 --- a/src/pages/recherche.py +++ b/src/pages/recherche.py @@ -1,3 +1,4 @@ +import dash_bootstrap_components as dbc from dash import Input, Output, State, callback, dcc, html, register_page from src.figures import DataTable @@ -77,7 +78,7 @@ layout = html.Div( # className="search_options", # children=[dcc.RadioItems(options=["Acheteur(s)"])], # ), - html.Div(id="search_results", className="wrapper"), + html.Div(id="search_results"), ], ) @@ -92,7 +93,7 @@ layout = html.Div( ) def update_search_results(n_submit, n_clicks, query): if query and len(query) >= 1: - content = [] + cols = [] for org_type in ["acheteur", "titulaire"]: if org_type == "acheteur": @@ -109,9 +110,8 @@ def update_search_results(n_submit, n_clicks, query): # Format output columns, tooltip = setup_table_columns(results, hideable=False) - org_content = [ + col_content = ( html.Div( - className=f"results_{org_type}", children=[ html.H3(f"{org_type.title()}s : {count}"), DataTable( @@ -125,10 +125,10 @@ def update_search_results(n_submit, n_clicks, query): ], ) if count > 0 - else html.P(f"Aucun {org_type} trouvé."), - ] - content.extend(org_content) - style = {"textAlign": "center", "display": "none"} + else html.P(f"Aucun {org_type} trouvé.") + ) + cols.append(dbc.Col(col_content, width=6)) - return content, style + style = {"textAlign": "center", "display": "none"} + return dbc.Row(cols), style return html.P(""), {"textAlign": "center"} diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 52ba25e..60b8977 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -82,68 +82,94 @@ layout = [ html.Div( children=[ html.Div( - className="wrapper", + style={"marginBottom": "50px"}, children=[ - html.H2( - className="org_title", + dbc.Row( + className="mb-2", children=[ - html.Span(id="titulaire_siret"), - " - ", - html.Span(id="titulaire_nom"), - ], - ), - html.Div( - className="org_year", - children=dcc.Dropdown( - id="titulaire_year", - options=["Toutes les années"] - + [ - str(year) - for year in range( - 2018, int(datetime.date.today().year) + 1 - ) - ], - placeholder="Année", - ), - ), - html.Div( - className="org_infos", - children=[ - # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. - html.P(["Commune : ", html.Strong(id="titulaire_commune")]), - html.P( - [ - "Département : ", - html.Strong(id="titulaire_departement"), - ] + dbc.Col( + html.H2( + children=[ + html.Span(id="titulaire_siret"), + " - ", + html.Span(id="titulaire_nom"), + ], + ), + width=8, ), - html.P(["Région : ", html.Strong(id="titulaire_region")]), - html.A( - id="titulaire_lien_annuaire", - children="Plus de détails sur l'Annuaire des entreprises", + dbc.Col( + dcc.Dropdown( + id="titulaire_year", + options=["Toutes les années"] + + [ + str(year) + for year in range( + 2018, int(datetime.date.today().year) + 1 + ) + ], + placeholder="Année", + ), + width=4, ), ], ), - html.Div( - className="org_stats", + dbc.Row( + className="mb-2", children=[ - html.P(id="titulaire_titre_stats"), - html.P(id="titulaire_marches_remportes"), - html.P(id="titulaire_acheteurs_differents"), - html.Button( - "Téléchargement au format Excel", - id="btn-download-data-titulaire", - className="btn btn-primary", + dbc.Col( + html.Div( + className="org_infos", + children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. + html.P(["Commune : ", html.Strong(id="titulaire_commune")]), + html.P( + [ + "Département : ", + html.Strong(id="titulaire_departement"), + ] + ), + html.P(["Région : ", html.Strong(id="titulaire_region")]), + html.A( + id="titulaire_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + ), + ], + ), + width=4, + ), + dbc.Col( + html.Div( + children=[ + html.P(id="titulaire_titre_stats"), + html.P(id="titulaire_marches_remportes"), + html.P(id="titulaire_acheteurs_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-data-titulaire", + className="btn btn-primary", + ), + dcc.Download(id="download-data-titulaire"), + ], + ), + width=4, + ), + dbc.Col( + html.Div(id="titulaire_map"), + width=4, ), - dcc.Download(id="download-data-titulaire"), ], ), - html.Div(className="org_map", id="titulaire_map"), - html.Div( - className="org_top", + dbc.Row( children=[ - html.H3("Top acheteurs"), - html.Div(className="marches_table", id="top10_acheteurs"), + dbc.Col( + html.Div( + children=[ + html.H3("Top acheteurs"), + html.Div(className="marches_table", id="top10_acheteurs"), + ], + ), + width=8, + ), ], ), html.Div(id="titulaire-distance-histogram"), From 0ac6c55d9e77091c9ad4f100c3f738884c6257a7 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 23:21:47 +0100 Subject: [PATCH 44/75] Correction imports --- CLAUDE.md | 4 ++++ src/callbacks.py | 2 +- src/pages/tableau.py | 5 ++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b4ead7d..b5483c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,10 @@ Tests require a running Chrome/Chromium browser. They use `DashComposite` from ` - `src/pages/*.py` — each page registers itself with `@register_page()` and owns its own layout and callbacks - `run.py` — dev entry point; exports `server` (Flask) for gunicorn +### Module imports + +- always import modules from the app starting with `src.` (e.g. `src.utils.`, `src.pages.recherche`, etc.) + ### Key pages | Page | URL | Purpose | diff --git a/src/callbacks.py b/src/callbacks.py index 9d791db..050a838 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -2,7 +2,7 @@ import polars as pl from dash import html from src.figures import DataTable -from utils import add_links_in_dict, format_values, setup_table_columns +from src.utils import add_links_in_dict, format_values, setup_table_columns def get_top_org_table(data, org_type: str, extra_columns: list): diff --git a/src/pages/tableau.py b/src/pages/tableau.py index 56fb9b8..4ea71c1 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -19,8 +19,7 @@ from dash import ( register_page, ) -from figures import make_column_picker -from src.figures import DataTable +from src.figures import DataTable, make_column_picker from src.utils import ( columns, df, @@ -31,8 +30,8 @@ from src.utils import ( meta_content, schema, sort_table_data, + prepare_table_data, ) -from utils import prepare_table_data update_date_timestamp = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date_timestamp).strftime("%d/%m/%Y") From d8f1a884a3e1c5a4e08b80df1ac0536a9c5cb919 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 23:35:04 +0100 Subject: [PATCH 45/75] fix: show actual km ranges in distance histogram tooltip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace px.histogram with manually-computed go.Bar so hover text displays human-readable distance ranges (e.g. "8.9 – 10.0 km") instead of raw log10 bin values. Co-Authored-By: Claude Sonnet 4.6 --- src/figures.py | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/figures.py b/src/figures.py index 14b624a..cab5b40 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,6 +1,7 @@ from typing import Literal from urllib.error import HTTPError, URLError +import numpy as np import dash_bootstrap_components as dbc import dash_leaflet as dl import dash_leaflet.express as dlx @@ -589,13 +590,42 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: .filter(pl.col("titulaire_distance") > 0) .collect(engine="streaming") ) - dff = dff.with_columns(pl.col("titulaire_distance").log(10)) - fig = px.histogram( - dff, - x="titulaire_distance", - nbins=50, - labels={"titulaire_distance": "Distance (km)"}, - ) + log_distances = dff["titulaire_distance"].log(10).to_numpy() + + fig = go.Figure() + if len(log_distances) > 0: + counts, bin_edges = np.histogram(log_distances, bins=50) + bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2 + bin_widths = bin_edges[1:] - bin_edges[:-1] + bin_edges_km = 10.0 ** bin_edges + + def fmt_km(km): + if km < 10: + return f"{km:.1f}" + elif km < 1000: + return f"{round(km)}" + else: + return f"{round(km):,}".replace(",", " ") + + hover_texts = [] + for i in range(len(counts)): + nb = f"{counts[i]:,}".replace(",", " ") + hover_texts.append( + f"Distance : {fmt_km(bin_edges_km[i])} – {fmt_km(bin_edges_km[i + 1])} km" + f"
Nombre de marchés : {nb}" + ) + + fig.add_trace( + go.Bar( + x=bin_centers, + y=counts, + width=bin_widths, + hovertext=hover_texts, + hoverinfo="text", + ) + ) + fig.update_layout(bargap=0) + fig.update_xaxes( tickvals=[0, 1, 2, 3, 4], ticktext=["1", "10", "100", "1 000", "10 000"], From dbfc20921ace4501ed95eef1acf1ee107d269891 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 23:54:41 +0100 Subject: [PATCH 46/75] =?UTF-8?q?Am=C3=A9lioration=20du=20layout=20dans=20?= =?UTF-8?q?titulaire=20et=20acheteur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 82 ++++++++++++++++++------------------ src/pages/observatoire.py | 2 +- src/pages/titulaire.py | 88 +++++++++++++++++++++------------------ 3 files changed, 91 insertions(+), 81 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 9be863c..1315e2e 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -117,44 +117,47 @@ layout = [ className="mb-2", children=[ dbc.Col( - html.Div( - className="org_infos", - children=[ - # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. - html.P(["Commune : ", html.Strong(id="acheteur_commune")]), - html.P( - [ - "Département : ", - html.Strong(id="acheteur_departement"), - ] - ), - html.P(["Région : ", html.Strong(id="acheteur_region")]), - html.A( - id="acheteur_lien_annuaire", - children="Plus de détails sur l'Annuaire des entreprises", - ), - ], - ), + className="org_infos", + children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. + html.P( + [ + "Commune : ", + html.Strong(id="acheteur_commune"), + ] + ), + html.P( + [ + "Département : ", + html.Strong(id="acheteur_departement"), + ] + ), + html.P( + ["Région : ", html.Strong(id="acheteur_region")] + ), + html.A( + id="acheteur_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + ), + ], width=4, ), dbc.Col( - html.Div( - children=[ - html.P(id="acheteur_titre_stats"), - html.P(id="acheteur_marches_attribues"), - html.P(id="acheteur_titulaires_differents"), - html.Button( - "Téléchargement au format Excel", - id="btn-download-data-acheteur", - className="btn btn-primary", - ), - dcc.Download(id="download-data-acheteur"), - ], - ), + children=[ + html.P(id="acheteur_titre_stats"), + html.P(id="acheteur_marches_attribues"), + html.P(id="acheteur_titulaires_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-data-acheteur", + className="btn btn-primary", + ), + dcc.Download(id="download-data-acheteur"), + ], width=4, ), dbc.Col( - html.Div(id="acheteur_map"), + id="acheteur_map", width=4, ), ], @@ -162,17 +165,13 @@ layout = [ dbc.Row( children=[ dbc.Col( - html.Div( - children=[ - html.H3("Top titulaires"), - html.Div(className="marches_table", id="top10_titulaires"), - ], - ), + className="marches_table", + id="top10_titulaires", width=8, ), + dbc.Col(id="acheteur-distance-histogram", width=4), ], ), - html.Div(id="acheteur-distance-histogram"), ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -373,7 +372,8 @@ def get_last_marches_data( Input(component_id="acheteur_data", component_property="data"), ) def get_top_titulaires(data): - return get_top_org_table(data, "titulaire", ["titulaire_distance", "montant"]) + table = get_top_org_table(data, "titulaire", ["titulaire_distance", "montant"]) + return make_card(fig=table, title="Top titulaires", lg=12, xl=12) @callback( @@ -522,4 +522,6 @@ def update_acheteur_distance_histogram(data): title="Distance acheteur–titulaire", subtitle="en nombre de marchés, échelle logarithmique", fig=fig, + lg=12, + xl=12, ) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 52ccbd4..f453871 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -138,7 +138,7 @@ layout = [ [ dcc.Markdown( """ -Les données saisies et publiées par les acheteurs comportent de nombreux montants farfelus qui sabotent les statistiques, au lieu de montants estimés avec rigueur. On parle de montant atteignant parfois les millions de milliards. Certains réutilisateurs mettent de côté ces marchés ou bien modifient les montants selon des règles fatalement arbitraires. J'ai fait le choix de ne quasiment pas modifier les données* afin de visibiliser le problème. +Les données saisies et publiées par les acheteurs comportent de nombreux montants farfelus qui sabotent les statistiques, au lieu de montants estimés avec rigueur. On parle de montants atteignant parfois les millions de milliards. Certains réutilisateurs des données mettent de côté ces marchés ou bien modifient les montants selon des règles fatalement arbitraires. J'ai fait le choix de ne quasiment pas modifier les données* afin de visibiliser le problème. Alors, on fait comment ? diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 60b8977..158a298 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -19,7 +19,6 @@ from src.callbacks import get_top_org_table from src.figures import ( DataTable, get_distance_histogram, - make_card, make_column_picker, point_on_map, ) @@ -117,44 +116,50 @@ layout = [ className="mb-2", children=[ dbc.Col( - html.Div( - className="org_infos", - children=[ - # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. - html.P(["Commune : ", html.Strong(id="titulaire_commune")]), - html.P( - [ - "Département : ", - html.Strong(id="titulaire_departement"), - ] - ), - html.P(["Région : ", html.Strong(id="titulaire_region")]), - html.A( - id="titulaire_lien_annuaire", - children="Plus de détails sur l'Annuaire des entreprises", - ), - ], - ), + className="org_infos", + children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. + html.P( + [ + "Commune : ", + html.Strong(id="titulaire_commune"), + ] + ), + html.P( + [ + "Département : ", + html.Strong(id="titulaire_departement"), + ] + ), + html.P( + [ + "Région : ", + html.Strong(id="titulaire_region"), + ] + ), + html.A( + id="titulaire_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + ), + ], width=4, ), dbc.Col( - html.Div( - children=[ - html.P(id="titulaire_titre_stats"), - html.P(id="titulaire_marches_remportes"), - html.P(id="titulaire_acheteurs_differents"), - html.Button( - "Téléchargement au format Excel", - id="btn-download-data-titulaire", - className="btn btn-primary", - ), - dcc.Download(id="download-data-titulaire"), - ], - ), + children=[ + html.P(id="titulaire_titre_stats"), + html.P(id="titulaire_marches_remportes"), + html.P(id="titulaire_acheteurs_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-data-titulaire", + className="btn btn-primary", + ), + dcc.Download(id="download-data-titulaire"), + ], width=4, ), dbc.Col( - html.Div(id="titulaire_map"), + id="titulaire_map", width=4, ), ], @@ -165,14 +170,17 @@ layout = [ html.Div( children=[ html.H3("Top acheteurs"), - html.Div(className="marches_table", id="top10_acheteurs"), + html.Div( + className="marches_table", + id="top10_acheteurs", + ), ], ), width=8, ), + dbc.Col(id="titulaire-distance-histogram", width=4), ], ), - html.Div(id="titulaire-distance-histogram"), ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -536,8 +544,8 @@ def update_titulaire_distance_histogram(data): pl.col("titulaire_distance").cast(pl.Float64, strict=False) ) fig = get_distance_histogram(lff) - return make_card( - title="Distance acheteur–titulaire", - subtitle="en nombre de marchés, échelle logarithmique", - fig=fig, - ) + return [ + html.H3("Distance acheteur-titulaire"), + html.H6("par nombre de marchés", className="card-subtitle mb-2 text-muted"), + fig, + ] From a9d3d96a5e48d8d196c4900217bbb2f8d7fe1bf3 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 19 Mar 2026 00:04:29 +0100 Subject: [PATCH 47/75] =?UTF-8?q?"Partager"=20masqu=C3=A9=20dans=20l'obser?= =?UTF-8?q?vatoire=20pour=20l'instant=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index f453871..b1d8cb0 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -479,6 +479,7 @@ def sync_observatoire_share_url(acheteur_id, titulaire_id, href): "Partager", className="btn btn-primary mt-2", title="Copier l'adresse de cette vue filtrée pour la partager.", + style={"display": "none"}, ) ], ) From 144e7142358083cf7cc570718ecc90d98239d506 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 19 Mar 2026 12:54:12 +0100 Subject: [PATCH 48/75] fix: reduce right margin in distance histogram to use full card width Co-Authored-By: Claude Sonnet 4.6 --- src/figures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/figures.py b/src/figures.py index cab5b40..415d708 100644 --- a/src/figures.py +++ b/src/figures.py @@ -626,6 +626,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: ) fig.update_layout(bargap=0) + fig.update_layout(margin=dict(r=10)) fig.update_xaxes( tickvals=[0, 1, 2, 3, 4], ticktext=["1", "10", "100", "1 000", "10 000"], From 9852d55a3d9735ac485ae029dc7faed78bed7907 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 19 Mar 2026 13:01:51 +0100 Subject: [PATCH 49/75] =?UTF-8?q?Tentative=20de=20tri=20des=20d=C3=A9parte?= =?UTF-8?q?metns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index b1d8cb0..ba32e9d 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -51,9 +51,13 @@ for year in reversed(range(2017, datetime.now().year + 1)): year = str(year) options_years[year] = year -options_departements = {} -for code, obj in departements.items(): - options_departements[code] = f"{obj['departement']} ({code})" +options_departements = { + code: f"{departements[code]['departement']} ({code})" + for code in sorted( + departements, + key=lambda c: 20.1 if c == "2A" else (20.2 if c == "2B" else float(c)), + ) +} def _apply_filters( From 50c3947c041bccd6c48eb1c828ccf1299a852fb7 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 19 Mar 2026 18:36:26 +0100 Subject: [PATCH 50/75] Filtres sur techniques, sous-traitance, innovant #65 --- src/assets/css/style.css | 4 +- src/pages/observatoire.py | 405 ++++++++++++++++++-------------------- src/utils.py | 91 +++++++++ 3 files changed, 289 insertions(+), 211 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index d76b6bb..42a4def 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -94,7 +94,6 @@ button:hover:not([disabled]) { padding: 28px 24px 0 24px; } - #header > * { margin: 0 0 20px 0px; } @@ -174,7 +173,6 @@ p.version > a { margin-right: 12px; } - /* --- Dashboard inputs --- */ .Select--multi .Select-value { @@ -182,7 +180,7 @@ p.version > a { background-color: rgba(255, 240, 240, 0.4); } -#filters .col > * { +#filters .row > * { margin-bottom: 6px; } diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index ba32e9d..77c32fb 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -1,5 +1,5 @@ import urllib.parse -from datetime import datetime, timedelta +from datetime import datetime import dash_bootstrap_components as dbc import polars as pl @@ -33,6 +33,7 @@ from src.utils import ( format_number, get_enum_values_as_dict, meta_content, + prepare_dashboard_data, ) name = "Observatoire" @@ -60,78 +61,6 @@ options_departements = { } -def _apply_filters( - lff: pl.LazyFrame, - year, - acheteur_id, - acheteur_categorie, - acheteur_departement_code, - titulaire_id, - titulaire_categorie, - titulaire_departement_code, - marche_type, - considerations_sociales, - considerations_environnementales, - montant_min=None, - montant_max=None, -) -> pl.LazyFrame: - if year: - lff = lff.filter(pl.col("dateNotification").dt.year() == int(year)) - else: - lff = lff.filter( - pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) - ) - - if acheteur_id: - lff = lff.filter(pl.col("acheteur_id").str.contains(acheteur_id)) - else: - if acheteur_categorie: - lff = lff.filter(pl.col("acheteur_categorie") == acheteur_categorie) - if acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in(acheteur_departement_code) - ) - - if titulaire_id: - lff = lff.filter(pl.col("titulaire_id").str.contains(titulaire_id)) - else: - if titulaire_categorie: - lff = lff.filter(pl.col("titulaire_categorie") == titulaire_categorie) - if titulaire_departement_code: - lff = lff.filter( - pl.col("titulaire_departement_code").is_in(titulaire_departement_code) - ) - - if marche_type: - lff = lff.filter(pl.col("type") == marche_type) - - if considerations_sociales: - lff = lff.filter( - pl.col("considerationsSociales") - .str.split(", ") - .list.set_intersection(considerations_sociales) - .list.len() - > 0 - ) - - if considerations_environnementales: - lff = lff.filter( - pl.col("considerationsEnvironnementales") - .str.split(", ") - .list.set_intersection(considerations_environnementales) - .list.len() - > 0 - ) - - if montant_min is not None: - lff = lff.filter(pl.col("montant") >= montant_min) - - if montant_max is not None: - lff = lff.filter(pl.col("montant") <= montant_max) - - return lff - - layout = [ dcc.Location(id="dashboard_url", refresh="callback-nav"), dcc.Store(id="observatoire-filters", storage_type="local"), @@ -181,6 +110,8 @@ Alors, on fait comment ? id="dashboard_year", options=options_years, placeholder="12 derniers mois", + persistence=True, + persistence_type="local", ), ), ), @@ -192,6 +123,8 @@ Alors, on fait comment ? placeholder="SIRET", debounce=True, style={"width": "100%"}, + persistence=True, + persistence_type="local", ), ), ), @@ -203,6 +136,8 @@ Alors, on fait comment ? "acheteur_categorie" ), placeholder="Catégorie", + persistence=True, + persistence_type="local", ) ), ), @@ -214,6 +149,8 @@ Alors, on fait comment ? multi=True, placeholder="Département", options=options_departements, + persistence=True, + persistence_type="local", ), ), ), @@ -225,6 +162,8 @@ Alors, on fait comment ? placeholder="SIRET", debounce=True, style={"width": "100%"}, + persistence=True, + persistence_type="local", ), ), ), @@ -236,6 +175,8 @@ Alors, on fait comment ? options=get_enum_values_as_dict( "titulaire_categorie" ), + persistence=True, + persistence_type="local", ), ), ), @@ -247,6 +188,8 @@ Alors, on fait comment ? multi=True, placeholder="Département", options=options_departements, + persistence=True, + persistence_type="local", ), ), ), @@ -257,30 +200,8 @@ Alors, on fait comment ? id="dashboard_marche_type", placeholder="Type", options=get_enum_values_as_dict("type"), - ), - ), - ), - dbc.Row( - dbc.Col( - dcc.Dropdown( - id="dashboard_marche_considerationsSociales", - placeholder="Considérations sociales", - options=get_enum_values_as_dict( - "considerationsSociales" - ), - multi=True, - ), - ), - ), - dbc.Row( - dbc.Col( - dcc.Dropdown( - id="dashboard_marche_considerationsEnvironnementales", - placeholder="Considérations environnementales", - multi=True, - options=get_enum_values_as_dict( - "considerationsEnvironnementales" - ), + persistence=True, + persistence_type="local", ), ), ), @@ -294,6 +215,8 @@ Alors, on fait comment ? min=0, debounce=True, style={"width": "100%"}, + persistence=True, + persistence_type="local", ), width=6, ), @@ -305,11 +228,113 @@ Alors, on fait comment ? min=0, debounce=True, style={"width": "100%"}, + persistence=True, + persistence_type="local", ), width=6, ), ] ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_techniques", + placeholder="Techniques d'achat", + options=get_enum_values_as_dict( + "techniques" + ), + multi=True, + persistence=True, + persistence_type="local", + ), + ), + ), + dbc.Row( + [ + dbc.Col("Sous-traitance :", lg=5), + dbc.Col( + dbc.RadioItems( + id="dashboard_marche_sousTraitanceDeclaree", + options=[ + { + "label": "Tous", + "value": "all", + }, + { + "label": "Oui", + "value": "oui", + }, + { + "label": "Non", + "value": "non", + }, + ], + value="all", + inline=True, + persistence=True, + persistence_type="local", + ), + lg=7, + ), + ] + ), + dbc.Row( + [ + dbc.Col("Marché innovant :", lg=5), + dbc.Col( + dbc.RadioItems( + id="dashboard_marche_innovant", + options=[ + { + "label": "Tous", + "value": "all", + }, + { + "label": "Oui", + "value": "oui", + }, + { + "label": "Non", + "value": "non", + }, + ], + value="all", + inline=True, + persistence=True, + persistence_type="local", + ), + lg=7, + ), + ] + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_considerationsSociales", + placeholder="Considérations sociales", + options=get_enum_values_as_dict( + "considerationsSociales" + ), + multi=True, + persistence=True, + persistence_type="local", + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_considerationsEnvironnementales", + placeholder="Considérations environnementales", + multi=True, + options=get_enum_values_as_dict( + "considerationsEnvironnementales" + ), + persistence=True, + persistence_type="local", + ), + ), + ), dcc.Download(id="download-observatoire"), dbc.Button( "Télécharger au format Excel", @@ -350,10 +375,13 @@ Alors, on fait comment ? Output("dashboard_titulaire_categorie", "value"), Output("dashboard_titulaire_departement_code", "value"), Output("dashboard_marche_type", "value"), - Output("dashboard_marche_considerationsSociales", "value"), - Output("dashboard_marche_considerationsEnvironnementales", "value"), Output("dashboard_montant_min", "value"), Output("dashboard_montant_max", "value"), + Output("dashboard_marche_techniques", "value"), + Output("dashboard_marche_innovant", "value"), + Output("dashboard_marche_sousTraitanceDeclaree", "value"), + Output("dashboard_marche_considerationsSociales", "value"), + Output("dashboard_marche_considerationsEnvironnementales", "value"), Input("dashboard_url", "search"), Input("dashboard_url", "pathname"), State("observatoire-filters", "data"), @@ -377,71 +405,11 @@ def restore_filters(search, _pathname, stored_filters): None, None, None, + None, + None, + None, ) - - if stored_filters: - return ( - stored_filters.get("year"), - stored_filters.get("acheteur_id"), - stored_filters.get("acheteur_categorie"), - stored_filters.get("acheteur_departement_code"), - stored_filters.get("titulaire_id"), - stored_filters.get("titulaire_categorie"), - stored_filters.get("titulaire_departement_code"), - stored_filters.get("marche_type"), - stored_filters.get("considerations_sociales"), - stored_filters.get("considerations_environnementales"), - stored_filters.get("montant_min"), - stored_filters.get("montant_max"), - ) - - return (no_update,) * 12 - - -@callback( - Output("observatoire-filters", "data"), - Input("dashboard_year", "value"), - Input("dashboard_acheteur_id", "value"), - Input("dashboard_acheteur_categorie", "value"), - Input("dashboard_acheteur_departement_code", "value"), - Input("dashboard_titulaire_id", "value"), - Input("dashboard_titulaire_categorie", "value"), - Input("dashboard_titulaire_departement_code", "value"), - Input("dashboard_marche_type", "value"), - Input("dashboard_marche_considerationsSociales", "value"), - Input("dashboard_marche_considerationsEnvironnementales", "value"), - Input("dashboard_montant_min", "value"), - Input("dashboard_montant_max", "value"), - prevent_initial_call=True, -) -def save_filters_to_storage( - year, - acheteur_id, - acheteur_categorie, - acheteur_departement_code, - titulaire_id, - titulaire_categorie, - titulaire_departement_code, - marche_type, - considerations_sociales, - considerations_environnementales, - montant_min, - montant_max, -): - return { - "year": year, - "acheteur_id": acheteur_id, - "acheteur_categorie": acheteur_categorie, - "acheteur_departement_code": acheteur_departement_code, - "titulaire_id": titulaire_id, - "titulaire_categorie": titulaire_categorie, - "titulaire_departement_code": titulaire_departement_code, - "marche_type": marche_type, - "considerations_sociales": considerations_sociales, - "considerations_environnementales": considerations_environnementales, - "montant_min": montant_min, - "montant_max": montant_max, - } + return (no_update,) * 15 @callback( @@ -503,10 +471,13 @@ def sync_observatoire_share_url(acheteur_id, titulaire_id, href): Input("dashboard_titulaire_categorie", "value"), Input("dashboard_titulaire_departement_code", "value"), Input("dashboard_marche_type", "value"), - Input("dashboard_marche_considerationsSociales", "value"), - Input("dashboard_marche_considerationsEnvironnementales", "value"), Input("dashboard_montant_min", "value"), Input("dashboard_montant_max", "value"), + Input("dashboard_marche_techniques", "value"), + Input("dashboard_marche_innovant", "value"), + Input("dashboard_marche_sousTraitanceDeclaree", "value"), + Input("dashboard_marche_considerationsSociales", "value"), + Input("dashboard_marche_considerationsEnvironnementales", "value"), ) def udpate_dashboard_cards( dashboard_year, @@ -517,10 +488,13 @@ def udpate_dashboard_cards( dashboard_titulaire_categorie, dashboard_titulaire_departement_code, dashboard_marche_type, - dashboard_marche_considerations_sociales, - dashboard_marche_considerations_environnementales, dashboard_montant_min, dashboard_montant_max, + dashboard_marche_techniques, + dashboard_marche_innovant, + dashboard_marche_sous_traitance_declaree, + dashboard_marche_considerations_sociales, + dashboard_marche_considerations_environnementales, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -531,23 +505,29 @@ def udpate_dashboard_cards( "montant", "considerationsSociales", "considerationsEnvironnementales", + "marcheInnovant", + "sousTraitanceDeclaree", + "techniques", "sourceDataset", "type", ) - lff = _apply_filters( - lff, - dashboard_year, - dashboard_acheteur_id, - dashboard_acheteur_categorie, - dashboard_acheteur_departement_code, - dashboard_titulaire_id, - dashboard_titulaire_categorie, - dashboard_titulaire_departement_code, - dashboard_marche_type, - dashboard_marche_considerations_sociales, - dashboard_marche_considerations_environnementales, + lff = prepare_dashboard_data( + lff=lff, + year=dashboard_year, + acheteur_id=dashboard_acheteur_id, + acheteur_categorie=dashboard_acheteur_categorie, + acheteur_departement_code=dashboard_acheteur_departement_code, + titulaire_id=dashboard_titulaire_id, + titulaire_categorie=dashboard_titulaire_categorie, + titulaire_departement_code=dashboard_titulaire_departement_code, + type=dashboard_marche_type, + considerations_sociales=dashboard_marche_considerations_sociales, + considerations_environnementales=dashboard_marche_considerations_environnementales, montant_min=dashboard_montant_min, montant_max=dashboard_montant_max, + techniques=dashboard_marche_techniques, + marche_innovant=dashboard_marche_innovant, + sous_traitance_declaree=dashboard_marche_sous_traitance_declaree, ) # Génération des métriques @@ -620,7 +600,7 @@ def udpate_dashboard_cards( cards.append( make_card( title="Catégorie d'entreprise", - subtitle="en nombre de marchés attribués", + subtitle="en nombre de titulaires", fig=donut_titulaire_categorie, ) ) @@ -683,41 +663,50 @@ def udpate_dashboard_cards( State("dashboard_titulaire_categorie", "value"), State("dashboard_titulaire_departement_code", "value"), State("dashboard_marche_type", "value"), - State("dashboard_marche_considerationsSociales", "value"), - State("dashboard_marche_considerationsEnvironnementales", "value"), State("dashboard_montant_min", "value"), State("dashboard_montant_max", "value"), + State("dashboard_marche_techniques", "value"), + State("dashboard_marche_innovant", "value"), + State("dashboard_marche_sousTraitanceDeclaree", "value"), + State("dashboard_marche_considerationsSociales", "value"), + State("dashboard_marche_considerationsEnvironnementales", "value"), prevent_initial_call=True, ) def download_observatoire( _n_clicks, - year, - acheteur_id, - acheteur_categorie, - acheteur_departement_code, - titulaire_id, - titulaire_categorie, - titulaire_departement_code, - marche_type, - considerations_sociales, - considerations_environnementales, - montant_min, - montant_max, + dashboard_year, + dashboard_acheteur_id, + dashboard_acheteur_categorie, + dashboard_acheteur_departement_code, + dashboard_titulaire_id, + dashboard_titulaire_categorie, + dashboard_titulaire_departement_code, + dashboard_marche_type, + dashboard_montant_min, + dashboard_montant_max, + dashboard_marche_techniques, + dashboard_marche_innovant, + dashboard_marche_sous_traitance_declaree, + dashboard_considerations_sociales, + dashboard_considerations_environnementales, ): - lff = _apply_filters( - df.lazy(), - year, - acheteur_id, - acheteur_categorie, - acheteur_departement_code, - titulaire_id, - titulaire_categorie, - titulaire_departement_code, - marche_type, - considerations_sociales, - considerations_environnementales, - montant_min=montant_min, - montant_max=montant_max, + lff = prepare_dashboard_data( + lff=df.lazy(), + year=dashboard_year, + acheteur_id=dashboard_acheteur_id, + acheteur_categorie=dashboard_acheteur_categorie, + acheteur_departement_code=dashboard_acheteur_departement_code, + titulaire_id=dashboard_titulaire_id, + titulaire_categorie=dashboard_titulaire_categorie, + titulaire_departement_code=dashboard_titulaire_departement_code, + type=dashboard_marche_type, + considerations_sociales=dashboard_considerations_sociales, + considerations_environnementales=dashboard_considerations_environnementales, + montant_min=dashboard_montant_min, + montant_max=dashboard_montant_max, + techniques=dashboard_marche_techniques, + marche_innovant=dashboard_marche_innovant, + sous_traitance_declaree=dashboard_marche_sous_traitance_declaree, ) def to_bytes(buffer): diff --git a/src/utils.py b/src/utils.py index d791cf9..19a7d70 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,6 +3,7 @@ import logging import os import uuid from collections import OrderedDict +from datetime import datetime, timedelta from time import localtime, sleep import polars as pl @@ -694,6 +695,96 @@ def prepare_table_data( ) +def prepare_dashboard_data( + lff: pl.LazyFrame, + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + type, + considerations_sociales, + considerations_environnementales, + techniques, + marche_innovant, + sous_traitance_declaree, + montant_min=None, + montant_max=None, +) -> pl.LazyFrame: + if year: + lff = lff.filter(pl.col("dateNotification").dt.year() == int(year)) + else: + lff = lff.filter( + pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) + ) + + if acheteur_id: + lff = lff.filter(pl.col("acheteur_id").str.contains(acheteur_id)) + else: + if acheteur_categorie: + lff = lff.filter(pl.col("acheteur_categorie") == acheteur_categorie) + if acheteur_departement_code: + lff = lff.filter( + pl.col("acheteur_departement_code").is_in(acheteur_departement_code) + ) + + if titulaire_id: + lff = lff.filter(pl.col("titulaire_id").str.contains(titulaire_id)) + else: + if titulaire_categorie: + lff = lff.filter(pl.col("titulaire_categorie") == titulaire_categorie) + if titulaire_departement_code: + lff = lff.filter( + pl.col("titulaire_departement_code").is_in(titulaire_departement_code) + ) + + if type: + lff = lff.filter(pl.col("type") == type) + + if marche_innovant != "all": + lff = lff.filter(pl.col("marcheInnovant") == marche_innovant) + + if sous_traitance_declaree != "all": + lff = lff.filter(pl.col("sousTraitanceDeclaree") == sous_traitance_declaree) + + if techniques: + lff = lff.filter( + pl.col("techniques") + .str.split(", ") + .list.set_intersection(techniques) + .list.len() + > 0 + ) + + if considerations_sociales: + lff = lff.filter( + pl.col("considerationsSociales") + .str.split(", ") + .list.set_intersection(considerations_sociales) + .list.len() + > 0 + ) + + if considerations_environnementales: + lff = lff.filter( + pl.col("considerationsEnvironnementales") + .str.split(", ") + .list.set_intersection(considerations_environnementales) + .list.len() + > 0 + ) + + if montant_min is not None: + lff = lff.filter(pl.col("montant") >= montant_min) + + if montant_max is not None: + lff = lff.filter(pl.col("montant") <= montant_max) + + return lff + + def get_button_properties(height): if height > 65000: download_disabled = True From dac9efee88c4dde862c4cca957ddeefd7300370d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 10:20:44 +0100 Subject: [PATCH 51/75] =?UTF-8?q?Carte=20r=C3=A9sum=C3=A9=20=3D>=20figures?= =?UTF-8?q?.py=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 40 +++++++++++++++++++++++++++++++++++++-- src/pages/observatoire.py | 33 +++----------------------------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/figures.py b/src/figures.py index 415d708..11f5b0a 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,10 +1,10 @@ from typing import Literal from urllib.error import HTTPError, URLError -import numpy as np import dash_bootstrap_components as dbc import dash_leaflet as dl import dash_leaflet.express as dlx +import numpy as np import plotly.express as px import plotly.graph_objects as go import polars as pl @@ -597,7 +597,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: counts, bin_edges = np.histogram(log_distances, bins=50) bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2 bin_widths = bin_edges[1:] - bin_edges[:-1] - bin_edges_km = 10.0 ** bin_edges + bin_edges_km = 10.0**bin_edges def fmt_km(km): if km < 10: @@ -636,6 +636,42 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: return dcc.Graph(figure=fig) +def get_dashboard_summary_table(dff, dff_per_uid, nb_marches): + nb_acheteurs = dff.select("acheteur_id").n_unique() + nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() + total_montant = int(dff_per_uid.select(pl.col("montant").sum()).item()) + + summary_table = [ + html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), + html.P( + [ + "Nombre d'acheteurs uniques : ", + html.Strong(str(format_number(nb_acheteurs))), + ] + ), + html.P( + [ + "Nombre de titulaires uniques : ", + html.Strong(str(format_number(nb_titulaires))), + ] + ), + html.P( + [ + "Montant total (", + html.Span( + "?", + id={"type": "modal-trigger", "index": "montant"}, + style={"cursor": "pointer", "textDecoration": "underline dotted"}, + ), + ") : ", + html.Strong(format_number(total_montant) + " €"), + ] + ), + ] + + return summary_table + + def make_card( title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4 ) -> dbc.Col: diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 77c32fb..8c3082b 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -19,6 +19,7 @@ from dash import ( from src.figures import ( get_barchart_sources, + get_dashboard_summary_table, get_distance_histogram, get_duplicate_matrix, get_geographic_maps, @@ -30,7 +31,6 @@ from src.utils import ( df, df_acheteurs, df_titulaires, - format_number, get_enum_values_as_dict, meta_content, prepare_dashboard_data, @@ -532,16 +532,9 @@ def udpate_dashboard_cards( # Génération des métriques dff = lff.collect(engine="streaming") - - # À transformer en fonction - nb_acheteurs = dff.select("acheteur_id").n_unique() - nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() - df_per_uid = ( dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first()) ) - - total_montant = int(df_per_uid.select(pl.col("montant").sum()).item()) nb_marches = df_per_uid.height if nb_marches == 0: @@ -553,29 +546,9 @@ def udpate_dashboard_cards( cards = [] - card_basic_counts = [ - html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), - html.P( - ["Nombre d'acheteurs : ", html.Strong(str(format_number(nb_acheteurs)))] - ), - html.P( - ["Nombre de titulaires : ", html.Strong(str(format_number(nb_titulaires)))] - ), - html.P( - [ - "Montant total (", - html.Span( - "?", - id={"type": "modal-trigger", "index": "montant"}, - style={"cursor": "pointer", "textDecoration": "underline dotted"}, - ), - ") : ", - html.Strong(format_number(total_montant) + " €"), - ] - ), - ] + card_summary_table = get_dashboard_summary_table(dff, df_per_uid, nb_marches) - cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) + cards.append(make_card(title="Résumé", paragraphs=card_summary_table)) donut_acheteur_categorie, nb_acheteur_categories = make_donut( lff, From 771dcf0ea1e829e30de7deec07d1da9dd5abb081 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 11:32:50 +0100 Subject: [PATCH 52/75] Moins de barres dans l'histogramme distances #65 --- src/figures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/figures.py b/src/figures.py index 11f5b0a..a6bd1dd 100644 --- a/src/figures.py +++ b/src/figures.py @@ -594,7 +594,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: fig = go.Figure() if len(log_distances) > 0: - counts, bin_edges = np.histogram(log_distances, bins=50) + counts, bin_edges = np.histogram(log_distances, bins=25) bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2 bin_widths = bin_edges[1:] - bin_edges[:-1] bin_edges_km = 10.0**bin_edges From 28fdca2a068a22f370d30ea640f7c05a31bcf299 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 11:33:25 +0100 Subject: [PATCH 53/75] =?UTF-8?q?Tri=20des=20d=C3=A9partements=20pour=20dr?= =?UTF-8?q?opdown=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 8c3082b..00d5dc7 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -52,14 +52,13 @@ for year in reversed(range(2017, datetime.now().year + 1)): year = str(year) options_years[year] = year -options_departements = { - code: f"{departements[code]['departement']} ({code})" - for code in sorted( - departements, - key=lambda c: 20.1 if c == "2A" else (20.2 if c == "2B" else float(c)), - ) -} - +options_departements = [] +for code in departements.keys(): + departement = { + "label": f"{departements[code]['departement']} ({code})", + "value": code, + } + options_departements.append(departement) layout = [ dcc.Location(id="dashboard_url", refresh="callback-nav"), From 4f9f31c4c7a0b3ce6b9762c7cfb71e4f6d5658f6 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 11:46:56 +0100 Subject: [PATCH 54/75] Ajout des filtres objet et code CPV #65 --- src/pages/observatoire.py | 66 +++++++++++++++++++++++++++++++++++++-- src/utils.py | 8 +++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 00d5dc7..4953732 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -204,6 +204,41 @@ Alors, on fait comment ? ), ), ), + dbc.Row( + dbc.Col( + dcc.Input( + id="dashboard_marche_objet", + placeholder="Objet", + debounce=True, + style={"width": "100%"}, + persistence=True, + persistence_type="local", + ), + ), + ), + dbc.Row( + [ + dbc.Col( + dcc.Input( + id="dashboard_marche_code_cpv", + placeholder="Code CPV (début)", + debounce=True, + style={"width": "100%"}, + persistence=True, + persistence_type="local", + ), + lg=8, + ), + dbc.Col( + html.A( + "liste des codes", + href="https://cpvcodes.eu/fr", + target="_blank", + ), + lg=4, + ), + ] + ), dbc.Row( [ dbc.Col( @@ -374,6 +409,8 @@ Alors, on fait comment ? Output("dashboard_titulaire_categorie", "value"), Output("dashboard_titulaire_departement_code", "value"), Output("dashboard_marche_type", "value"), + Output("dashboard_marche_objet", "value"), + Output("dashboard_marche_code_cpv", "value"), Output("dashboard_montant_min", "value"), Output("dashboard_montant_max", "value"), Output("dashboard_marche_techniques", "value"), @@ -407,8 +444,10 @@ def restore_filters(search, _pathname, stored_filters): None, None, None, + None, + None, ) - return (no_update,) * 15 + return (no_update,) * 17 @callback( @@ -470,6 +509,8 @@ def sync_observatoire_share_url(acheteur_id, titulaire_id, href): Input("dashboard_titulaire_categorie", "value"), Input("dashboard_titulaire_departement_code", "value"), Input("dashboard_marche_type", "value"), + Input("dashboard_marche_objet", "value"), + Input("dashboard_marche_code_cpv", "value"), Input("dashboard_montant_min", "value"), Input("dashboard_montant_max", "value"), Input("dashboard_marche_techniques", "value"), @@ -487,6 +528,8 @@ def udpate_dashboard_cards( dashboard_titulaire_categorie, dashboard_titulaire_departement_code, dashboard_marche_type, + dashboard_marche_objet, + dashboard_marche_code_cpv, dashboard_montant_min, dashboard_montant_max, dashboard_marche_techniques, @@ -496,7 +539,8 @@ def udpate_dashboard_cards( dashboard_marche_considerations_environnementales, ): lff: pl.LazyFrame = df.lazy() - lff = lff.select( + + columns = [ "uid", cs.starts_with("acheteur"), cs.starts_with("titulaire"), @@ -509,7 +553,15 @@ def udpate_dashboard_cards( "techniques", "sourceDataset", "type", - ) + "codeCPV", + ] + + if dashboard_marche_objet: + columns.append("objet") + + lff = lff.select(columns) + + # Filtrage des données lff = prepare_dashboard_data( lff=lff, year=dashboard_year, @@ -520,6 +572,8 @@ def udpate_dashboard_cards( titulaire_categorie=dashboard_titulaire_categorie, titulaire_departement_code=dashboard_titulaire_departement_code, type=dashboard_marche_type, + objet=dashboard_marche_objet, + code_cpv=dashboard_marche_code_cpv, considerations_sociales=dashboard_marche_considerations_sociales, considerations_environnementales=dashboard_marche_considerations_environnementales, montant_min=dashboard_montant_min, @@ -635,6 +689,8 @@ def udpate_dashboard_cards( State("dashboard_titulaire_categorie", "value"), State("dashboard_titulaire_departement_code", "value"), State("dashboard_marche_type", "value"), + State("dashboard_marche_objet", "value"), + State("dashboard_marche_code_cpv", "value"), State("dashboard_montant_min", "value"), State("dashboard_montant_max", "value"), State("dashboard_marche_techniques", "value"), @@ -654,6 +710,8 @@ def download_observatoire( dashboard_titulaire_categorie, dashboard_titulaire_departement_code, dashboard_marche_type, + dashboard_marche_objet, + dashboard_marche_code_cpv, dashboard_montant_min, dashboard_montant_max, dashboard_marche_techniques, @@ -672,6 +730,8 @@ def download_observatoire( titulaire_categorie=dashboard_titulaire_categorie, titulaire_departement_code=dashboard_titulaire_departement_code, type=dashboard_marche_type, + objet=dashboard_marche_objet, + code_cpv=dashboard_marche_code_cpv, considerations_sociales=dashboard_considerations_sociales, considerations_environnementales=dashboard_considerations_environnementales, montant_min=dashboard_montant_min, diff --git a/src/utils.py b/src/utils.py index 19a7d70..15ef713 100644 --- a/src/utils.py +++ b/src/utils.py @@ -705,6 +705,8 @@ def prepare_dashboard_data( titulaire_categorie, titulaire_departement_code, type, + objet, + code_cpv, considerations_sociales, considerations_environnementales, techniques, @@ -743,6 +745,12 @@ def prepare_dashboard_data( if type: lff = lff.filter(pl.col("type") == type) + if objet: + lff = lff.filter(pl.col("objet").str.contains(f"(?i){objet}")) + + if code_cpv: + lff = lff.filter(pl.col("codeCPV").str.starts_with(code_cpv)) + if marche_innovant != "all": lff = lff.filter(pl.col("marcheInnovant") == marche_innovant) From e5419bab9c7cd374dfee8b1e1ddc84c7fb838375 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 13:57:44 +0100 Subject: [PATCH 55/75] =?UTF-8?q?Donut=20:=20ajout=20=C3=A0=20autres=20si?= =?UTF-8?q?=20moins=20de=201%=20#65;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/figures.py b/src/figures.py index a6bd1dd..f2d02a9 100644 --- a/src/figures.py +++ b/src/figures.py @@ -715,6 +715,15 @@ def make_donut( lff = lff.with_columns(pl.col(title).replace(None, pl.lit(nulls))) dff = lff.collect(engine="streaming") nb_names = dff[title].n_unique() + if nb_names > 5: + sum_values = dff["Nombre"].sum() + dff = dff.with_columns( + pl.when((pl.col("Nombre") / sum_values) < 0.01) + .then(pl.lit("Autres")) + .otherwise(pl.col(title)) + .alias(title) + ) + dff = dff.with_columns( pl.col("Nombre") .map_elements(format_number, return_dtype=pl.String) From a89677604bd10b73d470d36db093a41a0eea3bd2 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 14:20:44 +0100 Subject: [PATCH 56/75] =?UTF-8?q?M=C3=A9dian=20des=20distances=20titulaire?= =?UTF-8?q?-acheteur=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/figures.py b/src/figures.py index f2d02a9..f94d2c8 100644 --- a/src/figures.py +++ b/src/figures.py @@ -640,6 +640,7 @@ def get_dashboard_summary_table(dff, dff_per_uid, nb_marches): nb_acheteurs = dff.select("acheteur_id").n_unique() nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() total_montant = int(dff_per_uid.select(pl.col("montant").sum()).item()) + median_distance = dff.select(pl.median("titulaire_distance")).item() summary_table = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), @@ -667,6 +668,12 @@ def get_dashboard_summary_table(dff, dff_per_uid, nb_marches): html.Strong(format_number(total_montant) + " €"), ] ), + html.P( + [ + "Distance acheteur-titulaire médiane : ", + html.Strong(format_number(median_distance) + " km"), + ] + ), ] return summary_table @@ -715,14 +722,14 @@ def make_donut( lff = lff.with_columns(pl.col(title).replace(None, pl.lit(nulls))) dff = lff.collect(engine="streaming") nb_names = dff[title].n_unique() - if nb_names > 5: - sum_values = dff["Nombre"].sum() - dff = dff.with_columns( - pl.when((pl.col("Nombre") / sum_values) < 0.01) - .then(pl.lit("Autres")) - .otherwise(pl.col(title)) - .alias(title) - ) + + sum_values = dff["Nombre"].sum() + dff = dff.with_columns( + pl.when((pl.col("Nombre") / sum_values) < 0.01) + .then(pl.lit("Autres")) + .otherwise(pl.col(title)) + .alias(title) + ) dff = dff.with_columns( pl.col("Nombre") From 91ea9eccea404a9f03040ec8e0a2e5e9a788d918 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 16:44:29 +0100 Subject: [PATCH 57/75] Top 10 acheteurs et titulaires #65 --- src/callbacks.py | 36 --------------------- src/figures.py | 68 +++++++++++++++++++++++++++++++++------ src/pages/acheteur.py | 4 +-- src/pages/observatoire.py | 11 +++++++ src/pages/titulaire.py | 4 +-- 5 files changed, 73 insertions(+), 50 deletions(-) delete mode 100644 src/callbacks.py diff --git a/src/callbacks.py b/src/callbacks.py deleted file mode 100644 index 050a838..0000000 --- a/src/callbacks.py +++ /dev/null @@ -1,36 +0,0 @@ -import polars as pl -from dash import html - -from src.figures import DataTable -from src.utils import add_links_in_dict, format_values, setup_table_columns - - -def get_top_org_table(data, org_type: str, extra_columns: list): - dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) - if dff.height == 0: - return html.Div() - - extra_columns = [] if extra_columns is None else extra_columns - - dff = dff.select(["uid", f"{org_type}_id", f"{org_type}_nom"] + extra_columns) - dff_nb = dff.group_by( - f"{org_type}_id", f"{org_type}_nom", "titulaire_distance" - ).agg(pl.len().alias("Attributions"), pl.sum("montant").alias("montant")) - dff_nb = dff_nb.sort(by="montant", descending=True, nulls_last=True) - dff_nb = dff_nb.cast(pl.String) - dff_nb = dff_nb.fill_null("") - dff_nb = format_values(dff_nb) - columns, tooltip = setup_table_columns( - dff_nb, hideable=False, exclude=[f"{org_type}_id"], new_columns=["Attributions"] - ) - data = dff_nb.to_dicts() - data = add_links_in_dict(data, f"{org_type}") - - return DataTable( - dtid=f"top10_{org_type}", - data=data, - page_action="native", - page_size=10, - columns=columns, - tooltip_header=tooltip, - ) diff --git a/src/figures.py b/src/figures.py index f94d2c8..687b5c0 100644 --- a/src/figures.py +++ b/src/figures.py @@ -11,7 +11,14 @@ import polars as pl from dash import dash_table, dcc, html from dash_extensions.javascript import Namespace -from src.utils import data_schema, departements_geojson, df, format_number +from src.utils import ( + add_links, + data_schema, + departements_geojson, + df, + format_number, + setup_table_columns, +) def get_yearly_statistics(statistics, today_str) -> html.Div: @@ -356,15 +363,15 @@ def get_duplicate_matrix() -> dcc.Graph: def get_geographic_maps(dff: pl.DataFrame) -> list | None: """ - Génère les cartes géographiques pour la métropole et les DOM-TOM. + Génère les cartes géographiques pour l'hexagone et les DOM-TOM. """ regions: dict = { - "Métropole": { + "Hexagone": { "coordinates": [46.6, 2.2], "zoom_leaflet": 5, "zoom_chloropleth": 1, - "name": "Métropole", + "name": "Hexagone", }, "971": { "coordinates": [16.23, -61.55], @@ -400,7 +407,7 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: def make_map_data(region_code: str) -> tuple[list, str or None]: lff: pl.LazyFrame = dff.lazy() - if region_code == "Métropole": + if region_code == "Hexagone": lff = lff.filter( (pl.col("acheteur_departement_code").str.len_chars() == 2) & (pl.col("titulaire_departement_code").str.len_chars() == 2) @@ -418,8 +425,8 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: dfs = [] - if (code == "Métropole" and nb_marches > 30000) or ( - code != "Métropole" and nb_marches > 10000 + if (code == "Hexagone" and nb_marches > 30000) or ( + code != "Hexagone" and nb_marches > 10000 ): _map_type: str = "chloropleth" @@ -490,7 +497,7 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: else: raise ValueError(f"Map type '{map_type}' not recognised") - lg, xl = (12, 8) if code == "Métropole" else (6, 4) + lg, xl = (12, 8) if code == "Hexagone" else (6, 4) col = make_card(regions[code]["name"], fig=map_graph, lg=lg, xl=xl) cols.append(col) @@ -573,7 +580,7 @@ def make_clusters_map(region: dict) -> dl.Map: zoom=zoom, style={ "width": "100%", - "height": "400px" if name == "Métropole" else "300px", + "height": "400px" if name == "Hexagone" else "300px", }, id=f"map-{region_id}", ) @@ -626,7 +633,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: ) fig.update_layout(bargap=0) - fig.update_layout(margin=dict(r=10)) + fig.update_layout(margin=dict(r=10, t=10)) fig.update_xaxes( tickvals=[0, 1, 2, 3, 4], ticktext=["1", "10", "100", "1 000", "10 000"], @@ -807,3 +814,44 @@ def make_column_picker(page: str): ) return table + + +def get_top_org_table(data, org_type: str, extra_columns: list, filters: bool = True): + if isinstance(data, pl.LazyFrame): + lff = data + else: + lff = pl.LazyFrame(data, strict=False, infer_schema_length=5000) + + if org_type == "titulaire": + extra_columns.append("titulaire_typeIdentifiant") + columns = ["uid", f"{org_type}_id", f"{org_type}_nom"] + extra_columns + + lff = lff.select(columns) + lff = lff.group_by([f"{org_type}_id", f"{org_type}_nom"] + extra_columns).agg( + pl.len().alias("Attributions") + ) + lff = lff.sort(by="Attributions", descending=True, nulls_last=True) + lff = lff.cast(pl.String) + lff = lff.fill_null("") + + dff: pl.DataFrame = lff.collect(engine="streaming") + + if dff.height == 0: + return html.Div() + + columns, tooltip = setup_table_columns( + dff, hideable=False, exclude=[f"{org_type}_id"], new_columns=["Attributions"] + ) + dff = add_links(dff) + data = dff.to_dicts() + # data = add_links_in_dict(data, f"{org_type}") + + return DataTable( + dtid=f"top10_{org_type}", + data=data, + page_action="native", + page_size=10, + columns=columns, + tooltip_header=tooltip, + filter_action="native" if filters else "none", + ) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 1315e2e..623af4c 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -15,10 +15,10 @@ from dash import ( register_page, ) -from src.callbacks import get_top_org_table from src.figures import ( DataTable, get_distance_histogram, + get_top_org_table, make_card, make_column_picker, point_on_map, @@ -372,7 +372,7 @@ def get_last_marches_data( Input(component_id="acheteur_data", component_property="data"), ) def get_top_titulaires(data): - table = get_top_org_table(data, "titulaire", ["titulaire_distance", "montant"]) + table = get_top_org_table(data, "titulaire", ["titulaire_distance"]) return make_card(fig=table, title="Top titulaires", lg=12, xl=12) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 4953732..3b66447 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -23,6 +23,7 @@ from src.figures import ( get_distance_histogram, get_duplicate_matrix, get_geographic_maps, + get_top_org_table, make_card, make_donut, ) @@ -649,6 +650,16 @@ def udpate_dashboard_cards( ) ) + top_acheteurs = get_top_org_table( + lff, org_type="acheteur", filters=False, extra_columns=[] + ) + cards.append(make_card(title="Top acheteurs", fig=top_acheteurs, lg=12, xl=8)) + + top_titulaires = get_top_org_table( + lff, org_type="titulaire", filters=False, extra_columns=[] + ) + cards.append(make_card(title="Top titulaires", fig=top_titulaires, lg=12, xl=8)) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) other_cards = [] diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 158a298..b13438c 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -15,10 +15,10 @@ from dash import ( register_page, ) -from src.callbacks import get_top_org_table from src.figures import ( DataTable, get_distance_histogram, + get_top_org_table, make_column_picker, point_on_map, ) @@ -395,7 +395,7 @@ def get_last_marches_data( Input(component_id="titulaire_data", component_property="data"), ) def get_top_acheteurs(data): - return get_top_org_table(data, "acheteur", ["titulaire_distance", "montant"]) + return get_top_org_table(data, "acheteur", ["titulaire_distance"]) @callback( From 3b2cb15935734cd7858bc6598481ff5d6bd6eeaa Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 17:38:10 +0100 Subject: [PATCH 58/75] =?UTF-8?q?Pr=C3=A9visualisation=20basique=20des=20d?= =?UTF-8?q?onn=C3=A9es=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 210 +++++++++++++++++++++++++++++++++++++- src/utils.py | 4 +- 2 files changed, 209 insertions(+), 5 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 3b66447..525d5b3 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -18,6 +18,7 @@ from dash import ( ) from src.figures import ( + DataTable, get_barchart_sources, get_dashboard_summary_table, get_distance_histogram, @@ -25,14 +26,17 @@ from src.figures import ( get_geographic_maps, get_top_org_table, make_card, + make_column_picker, make_donut, ) from src.utils import ( + data_schema, departements, df, df_acheteurs, df_titulaires, get_enum_values_as_dict, + logger, meta_content, prepare_dashboard_data, ) @@ -61,6 +65,27 @@ for code in departements.keys(): } options_departements.append(departement) +OBSERVATOIRE_COLUMNS = [ + col + for col in df.columns + if col.startswith("acheteur") + or col.startswith("titulaire") + or col + in [ + "uid", + "dateNotification", + "montant", + "considerationsSociales", + "considerationsEnvironnementales", + "marcheInnovant", + "sousTraitanceDeclaree", + "techniques", + "sourceDataset", + "type", + "codeCPV", + ] +] + layout = [ dcc.Location(id="dashboard_url", refresh="callback-nav"), dcc.Store(id="observatoire-filters", storage_type="local"), @@ -372,10 +397,16 @@ Alors, on fait comment ? ), dcc.Download(id="download-observatoire"), dbc.Button( - "Télécharger au format Excel", - id="btn-download-observatoire", - disabled=True, + "Prévisualiser les données", + id="btn-observatoire-preview", className="mt-2", + color="primary", + outline=True, + ), + dcc.Input( + id="observatoire-share-url", + readOnly=True, + style={"display": "none"}, ), dcc.Input( id="observatoire-share-url", @@ -398,6 +429,83 @@ Alors, on fait comment ? ), ], ), + dbc.Offcanvas( + id="observatoire-preview", + title="Prévisualisation des données", + placement="bottom", + is_open=False, + scrollable=True, + style={"height": "75vh"}, + children=[ + # Header row: title + "Colonnes affichées" button + dbc.Row( + [ + dbc.Col( + html.Div( + className="table-menu", + children=[ + dbc.Button( + "Colonnes affichées", + id="observatoire-preview-columns-open", + size="sm", + color="secondary", + outline=True, + ), + dbc.Button( + "Télécharger au format Excel", + id="btn-download-observatoire", + disabled=True, + className="mt-2", + color="primary", + outline=True, + ), + ], + ), + width="auto", + ), + ], + className="mb-2 align-items-center", + ), + # Column picker modal + dbc.Modal( + [ + dbc.ModalHeader( + dbc.ModalTitle("Colonnes affichées dans la prévisualisation") + ), + dbc.ModalBody( + id="observatoire-preview-columns-body", + children=make_column_picker("observatoire_preview"), + ), + dbc.ModalFooter( + dbc.Button( + "Fermer", + id="observatoire-preview-columns-close", + className="ms-auto", + n_clicks=0, + ) + ), + ], + id="observatoire-preview-columns", + is_open=False, + fullscreen="md-down", + scrollable=True, + size="xl", + ), + # DataTable + html.Div( + className="marches_table", + children=DataTable( + dtid="observatoire-preview-table", + page_size=10, + page_action="native", + sort_action="native", + filter_action="native", + hidden_columns=[], + columns=[{"id": col, "name": col} for col in OBSERVATOIRE_COLUMNS], + ), + ), + ], + ), ] @@ -586,6 +694,9 @@ def udpate_dashboard_cards( # Génération des métriques dff = lff.collect(engine="streaming") + + logger.debug("Filter data: " + str(dff.height)) + df_per_uid = ( dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first()) ) @@ -795,3 +906,96 @@ def add_organization_name_in_title(acheteur_id, titulaire_id): html.Small(nom, className="text-muted d-block fw-normal fs-5"), ] return name + + +@callback( + Output("observatoire-preview", "is_open"), + Input("btn-observatoire-preview", "n_clicks"), + State("observatoire-preview", "is_open"), + prevent_initial_call=True, +) +def toggle_observatoire_preview(n_clicks, is_open): + return not is_open + + +@callback( + Output("observatoire-preview-table", "data"), + Output("observatoire-preview-table", "columns"), + Input("observatoire-preview", "is_open"), + State("dashboard_year", "value"), + State("dashboard_acheteur_id", "value"), + State("dashboard_acheteur_categorie", "value"), + State("dashboard_acheteur_departement_code", "value"), + State("dashboard_titulaire_id", "value"), + State("dashboard_titulaire_categorie", "value"), + State("dashboard_titulaire_departement_code", "value"), + State("dashboard_marche_type", "value"), + State("dashboard_marche_objet", "value"), + State("dashboard_marche_code_cpv", "value"), + State("dashboard_montant_min", "value"), + State("dashboard_montant_max", "value"), + State("dashboard_marche_techniques", "value"), + State("dashboard_marche_innovant", "value"), + State("dashboard_marche_sousTraitanceDeclaree", "value"), + State("dashboard_marche_considerationsSociales", "value"), + State("dashboard_marche_considerationsEnvironnementales", "value"), + prevent_initial_call=True, +) +def populate_preview_table( + is_open, + dashboard_year, + dashboard_acheteur_id, + dashboard_acheteur_categorie, + dashboard_acheteur_departement_code, + dashboard_titulaire_id, + dashboard_titulaire_categorie, + dashboard_titulaire_departement_code, + dashboard_marche_type, + dashboard_marche_objet, + dashboard_marche_code_cpv, + dashboard_montant_min, + dashboard_montant_max, + dashboard_marche_techniques, + dashboard_marche_innovant, + dashboard_marche_sous_traitance_declaree, + dashboard_marche_considerations_sociales, + dashboard_marche_considerations_environnementales, +): + if not is_open: + return no_update, no_update + + available_in_df = [col for col in OBSERVATOIRE_COLUMNS if col in df.columns] + lff = prepare_dashboard_data( + lff=df.lazy().select(available_in_df), + year=dashboard_year, + acheteur_id=dashboard_acheteur_id, + acheteur_categorie=dashboard_acheteur_categorie, + acheteur_departement_code=dashboard_acheteur_departement_code, + titulaire_id=dashboard_titulaire_id, + titulaire_categorie=dashboard_titulaire_categorie, + titulaire_departement_code=dashboard_titulaire_departement_code, + type=dashboard_marche_type, + objet=dashboard_marche_objet, + code_cpv=dashboard_marche_code_cpv, + considerations_sociales=dashboard_marche_considerations_sociales, + considerations_environnementales=dashboard_marche_considerations_environnementales, + montant_min=dashboard_montant_min, + montant_max=dashboard_montant_max, + techniques=dashboard_marche_techniques, + marche_innovant=dashboard_marche_innovant, + sous_traitance_declaree=dashboard_marche_sous_traitance_declaree, + ) + + dff = lff.collect(engine="streaming") + + table_data = dff.to_dicts() + table_columns = [ + { + "name": data_schema.get(col, {}).get("title", col), + "id": col, + "type": "text", + "format": {"nully": "N/A"}, + } + for col in available_in_df + ] + return table_data, table_columns diff --git a/src/utils.py b/src/utils.py index 15ef713..6c1c996 100644 --- a/src/utils.py +++ b/src/utils.py @@ -751,10 +751,10 @@ def prepare_dashboard_data( if code_cpv: lff = lff.filter(pl.col("codeCPV").str.starts_with(code_cpv)) - if marche_innovant != "all": + if marche_innovant and marche_innovant != "all": lff = lff.filter(pl.col("marcheInnovant") == marche_innovant) - if sous_traitance_declaree != "all": + if sous_traitance_declaree and sous_traitance_declaree != "all": lff = lff.filter(pl.col("sousTraitanceDeclaree") == sous_traitance_declaree) if techniques: From dd63feeeac50f650b1ec1fa48011a2a8876b1d90 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 20 Mar 2026 17:49:27 +0100 Subject: [PATCH 59/75] Style des boutons --- src/pages/observatoire.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 525d5b3..b02d66f 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -399,7 +399,7 @@ Alors, on fait comment ? dbc.Button( "Prévisualiser les données", id="btn-observatoire-preview", - className="mt-2", + className="btn btn-primary", color="primary", outline=True, ), @@ -447,16 +447,13 @@ Alors, on fait comment ? dbc.Button( "Colonnes affichées", id="observatoire-preview-columns-open", - size="sm", - color="secondary", - outline=True, + className="btn btn-primary", ), dbc.Button( "Télécharger au format Excel", id="btn-download-observatoire", disabled=True, - className="mt-2", - color="primary", + className="btn btn-primary", outline=True, ), ], From d6e14e25641f778231b156d0bfdcd333e6ea296c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 10:02:11 +0100 Subject: [PATCH 60/75] =?UTF-8?q?Pr=C3=A9visualisation=20des=20donn=C3=A9e?= =?UTF-8?q?s=20fonctionnelle=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 149 +++++++++++--------------------------- src/utils.py | 2 + 2 files changed, 43 insertions(+), 108 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index b02d66f..e60b273 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -3,7 +3,6 @@ from datetime import datetime import dash_bootstrap_components as dbc import polars as pl -import polars.selectors as cs from dash import ( ALL, Input, @@ -30,7 +29,6 @@ from src.figures import ( make_donut, ) from src.utils import ( - data_schema, departements, df, df_acheteurs, @@ -39,6 +37,7 @@ from src.utils import ( logger, meta_content, prepare_dashboard_data, + prepare_table_data, ) name = "Observatoire" @@ -86,9 +85,14 @@ OBSERVATOIRE_COLUMNS = [ ] ] +DF_FILTERED: pl.DataFrame = pl.DataFrame() + layout = [ dcc.Location(id="dashboard_url", refresh="callback-nav"), dcc.Store(id="observatoire-filters", storage_type="local"), + dcc.Store( + id="filter-cleanup-trigger-observatoire-preview" + ), # utilisé juste pour ne pas avoir à adapter les données retournées de prepare_table data dbc.Modal( [ dbc.ModalHeader(dbc.ModalTitle("Montants")), @@ -445,10 +449,11 @@ Alors, on fait comment ? className="table-menu", children=[ dbc.Button( - "Colonnes affichées", + "Choisir les colonnes", id="observatoire-preview-columns-open", className="btn btn-primary", ), + html.P(id="nb_rows_observatoire"), dbc.Button( "Télécharger au format Excel", id="btn-download-observatoire", @@ -493,10 +498,10 @@ Alors, on fait comment ? className="marches_table", children=DataTable( dtid="observatoire-preview-table", - page_size=10, - page_action="native", - sort_action="native", - filter_action="native", + page_size=5, + page_action="custom", + sort_action="custom", + filter_action="custom", hidden_columns=[], columns=[{"id": col, "name": col} for col in OBSERVATOIRE_COLUMNS], ), @@ -605,8 +610,6 @@ def sync_observatoire_share_url(acheteur_id, titulaire_id, href): @callback( Output("cards", "children"), - Output("btn-download-observatoire", "disabled"), - Output("btn-download-observatoire", "children"), Input("dashboard_year", "value"), Input("dashboard_acheteur_id", "value"), Input("dashboard_acheteur_categorie", "value"), @@ -646,27 +649,6 @@ def udpate_dashboard_cards( ): lff: pl.LazyFrame = df.lazy() - columns = [ - "uid", - cs.starts_with("acheteur"), - cs.starts_with("titulaire"), - "dateNotification", - "montant", - "considerationsSociales", - "considerationsEnvironnementales", - "marcheInnovant", - "sousTraitanceDeclaree", - "techniques", - "sourceDataset", - "type", - "codeCPV", - ] - - if dashboard_marche_objet: - columns.append("objet") - - lff = lff.select(columns) - # Filtrage des données lff = prepare_dashboard_data( lff=lff, @@ -692,6 +674,9 @@ def udpate_dashboard_cards( # Génération des métriques dff = lff.collect(engine="streaming") + global DF_FILTERED + DF_FILTERED = dff + logger.debug("Filter data: " + str(dff.height)) df_per_uid = ( @@ -699,13 +684,6 @@ def udpate_dashboard_cards( ) nb_marches = df_per_uid.height - if nb_marches == 0: - dl_disabled, dl_text = True, "Pas de données à télécharger" - elif nb_marches > 65000: - dl_disabled, dl_text = True, "Téléchargement désactivé au-delà de 65 000 lignes" - else: - dl_disabled, dl_text = False, "Télécharger au format Excel" - cards = [] card_summary_table = get_dashboard_summary_table(dff, df_per_uid, nb_marches) @@ -794,7 +772,7 @@ def udpate_dashboard_cards( ) ) - return dbc.Row(children=cards + geographic_maps + other_cards), dl_disabled, dl_text + return dbc.Row(children=cards + geographic_maps + other_cards) @callback( @@ -918,81 +896,36 @@ def toggle_observatoire_preview(n_clicks, is_open): @callback( Output("observatoire-preview-table", "data"), Output("observatoire-preview-table", "columns"), + Output("observatoire-preview-table", "tooltip_header"), + Output("observatoire-preview-table", "data_timestamp"), + Output("nb_rows_observatoire", "children"), + Output("btn-download-observatoire", "disabled"), + Output("btn-download-observatoire", "children"), + Output("btn-download-observatoire", "title"), + Output("filter-cleanup-trigger-observatoire-preview", "data", allow_duplicate=True), Input("observatoire-preview", "is_open"), - State("dashboard_year", "value"), - State("dashboard_acheteur_id", "value"), - State("dashboard_acheteur_categorie", "value"), - State("dashboard_acheteur_departement_code", "value"), - State("dashboard_titulaire_id", "value"), - State("dashboard_titulaire_categorie", "value"), - State("dashboard_titulaire_departement_code", "value"), - State("dashboard_marche_type", "value"), - State("dashboard_marche_objet", "value"), - State("dashboard_marche_code_cpv", "value"), - State("dashboard_montant_min", "value"), - State("dashboard_montant_max", "value"), - State("dashboard_marche_techniques", "value"), - State("dashboard_marche_innovant", "value"), - State("dashboard_marche_sousTraitanceDeclaree", "value"), - State("dashboard_marche_considerationsSociales", "value"), - State("dashboard_marche_considerationsEnvironnementales", "value"), + Input("observano_updatetoire-preview-table", "filter_query"), + Input("observatoire-preview-table", "page_current"), + Input("observatoire-preview-table", "page_size"), + Input("observatoire-preview-table", "sort_by"), + State("observatoire-preview-table", "data_timestamp"), prevent_initial_call=True, ) def populate_preview_table( - is_open, - dashboard_year, - dashboard_acheteur_id, - dashboard_acheteur_categorie, - dashboard_acheteur_departement_code, - dashboard_titulaire_id, - dashboard_titulaire_categorie, - dashboard_titulaire_departement_code, - dashboard_marche_type, - dashboard_marche_objet, - dashboard_marche_code_cpv, - dashboard_montant_min, - dashboard_montant_max, - dashboard_marche_techniques, - dashboard_marche_innovant, - dashboard_marche_sous_traitance_declaree, - dashboard_marche_considerations_sociales, - dashboard_marche_considerations_environnementales, + is_open, filter_query, page_current, page_size, sort_by, data_timestamp ): if not is_open: - return no_update, no_update + return (no_update,) * 9 - available_in_df = [col for col in OBSERVATOIRE_COLUMNS if col in df.columns] - lff = prepare_dashboard_data( - lff=df.lazy().select(available_in_df), - year=dashboard_year, - acheteur_id=dashboard_acheteur_id, - acheteur_categorie=dashboard_acheteur_categorie, - acheteur_departement_code=dashboard_acheteur_departement_code, - titulaire_id=dashboard_titulaire_id, - titulaire_categorie=dashboard_titulaire_categorie, - titulaire_departement_code=dashboard_titulaire_departement_code, - type=dashboard_marche_type, - objet=dashboard_marche_objet, - code_cpv=dashboard_marche_code_cpv, - considerations_sociales=dashboard_marche_considerations_sociales, - considerations_environnementales=dashboard_marche_considerations_environnementales, - montant_min=dashboard_montant_min, - montant_max=dashboard_montant_max, - techniques=dashboard_marche_techniques, - marche_innovant=dashboard_marche_innovant, - sous_traitance_declaree=dashboard_marche_sous_traitance_declaree, + global DF_FILTERED + lff = DF_FILTERED.lazy() + + return prepare_table_data( + lff, + data_timestamp, + filter_query, + page_current, + page_size, + sort_by, + "observatoire-preview", ) - - dff = lff.collect(engine="streaming") - - table_data = dff.to_dicts() - table_columns = [ - { - "name": data_schema.get(col, {}).get("title", col), - "id": col, - "type": "text", - "format": {"nully": "N/A"}, - } - for col in available_in_df - ] - return table_data, table_columns diff --git a/src/utils.py b/src/utils.py index 6c1c996..63e5750 100644 --- a/src/utils.py +++ b/src/utils.py @@ -631,6 +631,8 @@ def prepare_table_data( # Récupération des données if isinstance(data, list): lff: pl.LazyFrame = pl.LazyFrame(data, strict=False, infer_schema_length=5000) + elif isinstance(data, pl.LazyFrame): + lff = data else: lff: pl.LazyFrame = df.lazy() # start from the original data From b34f63f711ee04b9e931e991f7e49543690a4160 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 10:17:58 +0100 Subject: [PATCH 61/75] =?UTF-8?q?Pr=C3=A9visualisation=20des=20donn=C3=A9e?= =?UTF-8?q?s:=20choix=20des=20colonnes=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 59 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index e60b273..4e3dcb9 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -29,10 +29,12 @@ from src.figures import ( make_donut, ) from src.utils import ( + columns, departements, df, df_acheteurs, df_titulaires, + get_default_hidden_columns, get_enum_values_as_dict, logger, meta_content, @@ -90,6 +92,7 @@ DF_FILTERED: pl.DataFrame = pl.DataFrame() layout = [ dcc.Location(id="dashboard_url", refresh="callback-nav"), dcc.Store(id="observatoire-filters", storage_type="local"), + dcc.Store(id="observatoire-hidden-columns", storage_type="local"), dcc.Store( id="filter-cleanup-trigger-observatoire-preview" ), # utilisé juste pour ne pas avoir à adapter les données retournées de prepare_table data @@ -487,7 +490,7 @@ Alors, on fait comment ? ) ), ], - id="observatoire-preview-columns", + id="observatoire-preview-columns-modal", is_open=False, fullscreen="md-down", scrollable=True, @@ -904,7 +907,7 @@ def toggle_observatoire_preview(n_clicks, is_open): Output("btn-download-observatoire", "title"), Output("filter-cleanup-trigger-observatoire-preview", "data", allow_duplicate=True), Input("observatoire-preview", "is_open"), - Input("observano_updatetoire-preview-table", "filter_query"), + Input("observatoire-preview-table", "filter_query"), Input("observatoire-preview-table", "page_current"), Input("observatoire-preview-table", "page_size"), Input("observatoire-preview-table", "sort_by"), @@ -929,3 +932,55 @@ def populate_preview_table( sort_by, "observatoire-preview", ) + + +@callback( + Output("observatoire-hidden-columns", "data", allow_duplicate=True), + Input("observatoire_preview_column_list", "selected_rows"), + prevent_initial_call=True, +) +def update_hidden_columns_from_checkboxes(selected_columns): + if selected_columns: + selected_columns = [columns[i] for i in selected_columns] + hidden_columns = [col for col in columns if col not in selected_columns] + return hidden_columns + else: + return [] + + +@callback( + Output("observatoire-preview-table", "hidden_columns"), + Input( + "observatoire-hidden-columns", + "data", + ), +) +def store_hidden_columns(hidden_columns): + return hidden_columns + + +@callback( + Output("observatoire_preview_column_list", "selected_rows"), + Input("observatoire-preview-table", "hidden_columns"), + State( + "observatoire_preview_column_list", "selected_rows" + ), # pour éviter la boucle infinie +) +def update_checkboxes_from_hidden_columns(hidden_cols, current_checkboxes): + hidden_cols = hidden_cols or get_default_hidden_columns("tableau") + + # Show all columns that are NOT hidden + visible_cols = [columns.index(col) for col in columns if col not in hidden_cols] + return visible_cols + + +@callback( + Output("observatoire-preview-columns-modal", "is_open"), + Input("observatoire-preview-columns-open", "n_clicks"), + Input("observatoire-preview-columns-close", "n_clicks"), + State("observatoire-preview-columns-modal", "is_open"), +) +def toggle_tableau_columns(click_open, click_close, is_open): + if click_open or click_close: + return not is_open + return is_open From e6bf671f1657292590078ce68d098c6cbe82655f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 10:28:07 +0100 Subject: [PATCH 62/75] =?UTF-8?q?Pr=C3=A9visualisation=20des=20donn=C3=A9e?= =?UTF-8?q?s:=20bonnes=20donn=C3=A9es=20t=C3=A9l=C3=A9charg=C3=A9es=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 4e3dcb9..ff5a959 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -798,6 +798,7 @@ def udpate_dashboard_cards( State("dashboard_marche_sousTraitanceDeclaree", "value"), State("dashboard_marche_considerationsSociales", "value"), State("dashboard_marche_considerationsEnvironnementales", "value"), + State("observatoire-hidden-columns", "data"), prevent_initial_call=True, ) def download_observatoire( @@ -819,6 +820,7 @@ def download_observatoire( dashboard_marche_sous_traitance_declaree, dashboard_considerations_sociales, dashboard_considerations_environnementales, + hidden_columns, ): lff = prepare_dashboard_data( lff=df.lazy(), @@ -841,6 +843,9 @@ def download_observatoire( sous_traitance_declaree=dashboard_marche_sous_traitance_declaree, ) + if hidden_columns: + lff = lff.drop(hidden_columns) + def to_bytes(buffer): lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") From ff6b5d0d41c7062fcd72ebb33adebee388c097f0 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:15:26 +0100 Subject: [PATCH 63/75] Add design spec for full URL sharing on observatoire page --- ...21-observatoire-full-url-sharing-design.md | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-21-observatoire-full-url-sharing-design.md diff --git a/docs/superpowers/specs/2026-03-21-observatoire-full-url-sharing-design.md b/docs/superpowers/specs/2026-03-21-observatoire-full-url-sharing-design.md new file mode 100644 index 0000000..9f910df --- /dev/null +++ b/docs/superpowers/specs/2026-03-21-observatoire-full-url-sharing-design.md @@ -0,0 +1,117 @@ +# Observatoire: Full URL Sharing for All Filters + +## Problem + +The "Partager" button on `/observatoire` currently only encodes `acheteur_id` and `titulaire_id` in the shareable URL. The other 15 filter parameters are lost, so a shared link does not reproduce the sender's filtered view. + +## Goal + +Extend URL sharing so that **all 17 filter parameters** are encoded in the URL and restored when a recipient opens it. The recipient sees exactly what the sender intended — URL params replace all local filter state. + +## Approach + +Flat query parameters with short, readable keys. Multi-value filters use repeated keys (native to `urllib.parse`). Only non-default values appear in the URL. + +## URL Parameter Mapping + +| Component ID | URL key | Type | Default (omitted) | +|---|---|---|---| +| `dashboard_year` | `annee` | single | `None` | +| `dashboard_acheteur_id` | `acheteur_id` | single | `None` | +| `dashboard_acheteur_categorie` | `acheteur_cat` | single | `None` | +| `dashboard_acheteur_departement_code` | `acheteur_dept` | multi | `[]`/`None` | +| `dashboard_titulaire_id` | `titulaire_id` | single | `None` | +| `dashboard_titulaire_categorie` | `titulaire_cat` | single | `None` | +| `dashboard_titulaire_departement_code` | `titulaire_dept` | multi | `[]`/`None` | +| `dashboard_marche_type` | `type` | single | `None` | +| `dashboard_marche_objet` | `objet` | single | `None` | +| `dashboard_marche_code_cpv` | `cpv` | single | `None` | +| `dashboard_montant_min` | `montant_min` | single (number) | `None` | +| `dashboard_montant_max` | `montant_max` | single (number) | `None` | +| `dashboard_marche_techniques` | `techniques` | multi | `[]`/`None` | +| `dashboard_marche_innovant` | `innovant` | single | `"all"` | +| `dashboard_marche_sousTraitanceDeclaree` | `sous_traitance` | single | `"all"` | +| `dashboard_marche_considerationsSociales` | `social` | multi | `[]`/`None` | +| `dashboard_marche_considerationsEnvironnementales` | `env` | multi | `[]`/`None` | + +Example URL: +``` +/observatoire?annee=2024&acheteur_id=12345678901234&acheteur_dept=75&acheteur_dept=13&montant_min=10000&innovant=oui +``` + +## Data Structure + +A list of tuples defines the mapping, used by both callbacks to avoid scattered string literals: + +```python +FILTER_PARAMS = [ + # (component_id, url_key, is_multi, default_value) + ("dashboard_year", "annee", False, None), + ("dashboard_acheteur_id", "acheteur_id", False, None), + ("dashboard_acheteur_categorie", "acheteur_cat", False, None), + ("dashboard_acheteur_departement_code", "acheteur_dept", True, None), + ("dashboard_titulaire_id", "titulaire_id", False, None), + ("dashboard_titulaire_categorie", "titulaire_cat", False, None), + ("dashboard_titulaire_departement_code", "titulaire_dept", True, None), + ("dashboard_marche_type", "type", False, None), + ("dashboard_marche_objet", "objet", False, None), + ("dashboard_marche_code_cpv", "cpv", False, None), + ("dashboard_montant_min", "montant_min", False, None), + ("dashboard_montant_max", "montant_max", False, None), + ("dashboard_marche_techniques", "techniques", True, None), + ("dashboard_marche_innovant", "innovant", False, "all"), + ("dashboard_marche_sousTraitanceDeclaree", "sous_traitance", False, "all"), + ("dashboard_marche_considerationsSociales", "social", True, None), + ("dashboard_marche_considerationsEnvironnementales", "env", True, None), +] +``` + +## Callback Changes + +### 1. `sync_observatoire_share_url` (line 575) + +**Current:** Takes `acheteur_id` and `titulaire_id` as Inputs. + +**New:** Takes all 17 filter values as Inputs (same as `udpate_dashboard_cards`). Builds the URL using `FILTER_PARAMS`, skipping default values. Uses `urllib.parse.urlencode(params, doseq=True)` for multi-value params. + +### 2. `restore_filters` (line 539) + +**Current:** Extracts only `acheteur_id` and `titulaire_id` from URL. + +**New:** +- Iterates over `FILTER_PARAMS` to extract all values from `parse_qs` +- For multi-value params: reads the full list from `parse_qs` (returns lists natively) +- For number params (`montant_min`, `montant_max`): casts to `float` +- The guard condition changes from `if acheteur_id or titulaire_id` to "if any URL param is present" — this is necessary so URLs like `?annee=2024&montant_min=10000` (without an ID) work correctly +- When **any** URL param is present: returns explicit values for all 17 outputs — the URL value for params present, `None`/default for params absent. This ensures "URL replaces all" semantics. +- When **no** URL params are present: returns `(no_update,) * 17` (preserving local persistence) +- Radio buttons (`innovant`, `sous_traitance`): value from URL if present, otherwise `"all"` (their default) + +### 3. Layout bug fix + +Remove the duplicate `dcc.Input(id="observatoire-share-url")` (lines 413-422 — two identical elements). + +## Backward Compatibility + +Old URLs with only `?acheteur_id=...` or `?titulaire_id=...` continue to work — the new `restore_filters` will read those keys and reset all others to defaults, which is the same effective behavior as before. + +Links generated by `add_links()` in `src/utils.py` (used on search results to link to `/observatoire?acheteur_id=...`) are unaffected. + +## Test Changes + +### Fix broken test `test_010_observatoire_montant_filter` + +This test imports `_apply_filters` from `pages.observatoire`, which no longer exists (replaced by `prepare_dashboard_data` in `src/utils.py`). Fix: +- Replace import with `from src.utils import prepare_dashboard_data` +- Update the call to match `prepare_dashboard_data`'s signature: rename `marche_type` keyword to `type`, and add missing params `objet`, `code_cpv`, `techniques`, `marche_innovant`, `sous_traitance_declaree` (all as `None`) + +### New test: multi-param URL round-trip + +Add a test that navigates to `/observatoire?annee=2024&acheteur_id=&montant_min=10000` and verifies that: +- `dashboard_year` dropdown shows "2024" +- `dashboard_acheteur_id` input contains the test ID +- `dashboard_montant_min` input contains "10000" + +### Update existing tests + +Tests `test_006` and `test_007` validate `acheteur_id` round-trip. These should continue to pass without changes since `acheteur_id` keeps the same URL key. From 547accd7be9bb93f17083c465def43db029d73c6 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:25:54 +0100 Subject: [PATCH 64/75] fix: update test_010 to use prepare_dashboard_data instead of removed _apply_filters Co-Authored-By: Claude Sonnet 4.6 --- tests/test_main.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index a50d807..4d11f7b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -221,10 +221,7 @@ def test_010_observatoire_montant_filter(): import polars as pl - from pages.observatoire import _apply_filters - from src.app import ( - app, # noqa: F401 – instantiates the Dash app before register_page() calls - ) + from src.utils import prepare_dashboard_data data = pl.DataFrame( { @@ -235,7 +232,7 @@ def test_010_observatoire_montant_filter(): ) def apply(min_val=None, max_val=None): - return _apply_filters( + return prepare_dashboard_data( data.lazy(), year="2025", acheteur_id=None, @@ -244,9 +241,14 @@ def test_010_observatoire_montant_filter(): titulaire_id=None, titulaire_categorie=None, titulaire_departement_code=None, - marche_type=None, + type=None, + objet=None, + code_cpv=None, considerations_sociales=None, considerations_environnementales=None, + techniques=None, + marche_innovant=None, + sous_traitance_declaree=None, montant_min=min_val, montant_max=max_val, ).collect() From 3957ca1662723b25addfa8d9922fab1d0b90ca87 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:27:54 +0100 Subject: [PATCH 65/75] feat: restore_filters reads all 17 filter params from URL Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 62 +++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index ff5a959..a8ce93d 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -514,6 +514,28 @@ Alors, on fait comment ? ] +FILTER_PARAMS = [ + # (component_id, url_key, is_multi, default_value) + ("dashboard_year", "annee", False, None), + ("dashboard_acheteur_id", "acheteur_id", False, None), + ("dashboard_acheteur_categorie", "acheteur_cat", False, None), + ("dashboard_acheteur_departement_code", "acheteur_dept", True, None), + ("dashboard_titulaire_id", "titulaire_id", False, None), + ("dashboard_titulaire_categorie", "titulaire_cat", False, None), + ("dashboard_titulaire_departement_code", "titulaire_dept", True, None), + ("dashboard_marche_type", "type", False, None), + ("dashboard_marche_objet", "objet", False, None), + ("dashboard_marche_code_cpv", "cpv", False, None), + ("dashboard_montant_min", "montant_min", False, None), + ("dashboard_montant_max", "montant_max", False, None), + ("dashboard_marche_techniques", "techniques", True, None), + ("dashboard_marche_innovant", "innovant", False, "all"), + ("dashboard_marche_sousTraitanceDeclaree", "sous_traitance", False, "all"), + ("dashboard_marche_considerationsSociales", "social", True, None), + ("dashboard_marche_considerationsEnvironnementales", "env", True, None), +] + + @callback( Output("dashboard_year", "value"), Output("dashboard_acheteur_id", "value"), @@ -539,28 +561,24 @@ Alors, on fait comment ? def restore_filters(search, _pathname, stored_filters): if search: params = urllib.parse.parse_qs(search.lstrip("?")) - acheteur_id = (params.get("acheteur_id") or [None])[0] or None - titulaire_id = (params.get("titulaire_id") or [None])[0] or None - if acheteur_id or titulaire_id: - return ( - None, - acheteur_id, - None, - None, - titulaire_id, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ) + known_keys = {fp[1] for fp in FILTER_PARAMS} + if any(k in params for k in known_keys): + values = [] + for _comp_id, url_key, is_multi, default in FILTER_PARAMS: + if url_key in params: + if is_multi: + values.append(params[url_key]) + else: + raw = params[url_key][0] + if url_key in ("montant_min", "montant_max"): + try: + raw = float(raw) + except (ValueError, TypeError): + raw = None + values.append(raw) + else: + values.append(default) + return tuple(values) return (no_update,) * 17 From bb2cde2fc5e168926aa70417100d0802436ee772 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:31:47 +0100 Subject: [PATCH 66/75] feat: sync_observatoire_share_url encodes all 17 filter params Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index a8ce93d..ba837b8 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -585,22 +585,29 @@ def restore_filters(search, _pathname, stored_filters): @callback( Output("observatoire-share-url", "value"), Output("observatoire-copy-container", "children"), - Input("dashboard_acheteur_id", "value"), - Input("dashboard_titulaire_id", "value"), + *[Input(fp[0], "value") for fp in FILTER_PARAMS], State("dashboard_url", "href"), prevent_initial_call=True, ) -def sync_observatoire_share_url(acheteur_id, titulaire_id, href): +def sync_observatoire_share_url(*args): + # Last arg is href (State), rest are filter values + filter_values = args[:-1] + href = args[-1] + if not href: return no_update, no_update base_url = href.split("?")[0] - params = {} - if acheteur_id: - params["acheteur_id"] = acheteur_id - if titulaire_id: - params["titulaire_id"] = titulaire_id + params = [] + for (_, url_key, is_multi, default), value in zip(FILTER_PARAMS, filter_values): + if value is None or value == default or value == [] or value == "": + continue + if is_multi and isinstance(value, list): + for v in value: + params.append((url_key, v)) + else: + params.append((url_key, value)) query_string = urllib.parse.urlencode(params) full_url = f"{base_url}?{query_string}" if query_string else base_url From 139b820b6b5c699e1010615c77f461563634b01a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:32:43 +0100 Subject: [PATCH 67/75] fix: remove duplicate observatoire-share-url element in layout Co-Authored-By: Claude Opus 4.6 --- src/pages/observatoire.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index ba837b8..39bb1e6 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -415,11 +415,6 @@ Alors, on fait comment ? readOnly=True, style={"display": "none"}, ), - dcc.Input( - id="observatoire-share-url", - readOnly=True, - style={"display": "none"}, - ), html.Div(id="observatoire-copy-container"), ], ), From b06f3c91e0bd5e9f28b12c5ec2c544c7c3c63066 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:33:38 +0100 Subject: [PATCH 68/75] test: add multi-param URL round-trip test for observatoire Co-Authored-By: Claude Sonnet 4.6 --- tests/test_main.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 4d11f7b..8719e7b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -306,6 +306,36 @@ def test_009_observatoire_filter_persistence(dash_duo: DashComposite): ) +def test_011_observatoire_multi_param_url(dash_duo: DashComposite): + import time + + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate with multiple filter params + dash_duo.wait_for_page( + f"{dash_duo.server_url}/observatoire?annee=2024&acheteur_id=12345678901234&montant_min=10000" + ) + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + time.sleep(1) # Allow callback chain to complete + + # Verify acheteur_id input + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "12345678901234", ( + "acheteur_id input should be populated from URL param" + ) + + # Verify montant_min input + montant_input = dash_duo.find_element("#dashboard_montant_min") + montant_value = montant_input.get_attribute("value") + assert montant_value in ("10000", "10000.0"), ( + f"montant_min input should be populated from URL param, got: {montant_value}" + ) + + def test_get_distance_histogram_returns_graph(): import polars as pl from dash import dcc From eecd75ac42641449162ccbdcd545247181499ab7 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 11:55:48 +0100 Subject: [PATCH 69/75] =?UTF-8?q?Style=20des=20boutons=20Pr=C3=A9visualise?= =?UTF-8?q?r=20et=20Partager=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 39bb1e6..2dc7882 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -401,8 +401,10 @@ Alors, on fait comment ? persistence_type="local", ), ), - ), - dcc.Download(id="download-observatoire"), + ), +dbc.Row( + [dbc.Col([ + dcc.Download(id="download-observatoire"), dbc.Button( "Prévisualiser les données", id="btn-observatoire-preview", @@ -414,8 +416,10 @@ Alors, on fait comment ? id="observatoire-share-url", readOnly=True, style={"display": "none"}, - ), - html.Div(id="observatoire-copy-container"), + )], lg=12, xl=7)]), +dbc.Row( +dbc.Col( + html.Div(id="observatoire-copy-container"), lg=12, xl=5)) ], ), dbc.Col( @@ -620,10 +624,9 @@ def sync_observatoire_share_url(*args): className="fa fa-link", children=[ dbc.Button( - "Partager", + "Partager cette vue", className="btn btn-primary mt-2", title="Copier l'adresse de cette vue filtrée pour la partager.", - style={"display": "none"}, ) ], ) From 7b78e0a0eac589629b5dd7a0539dde104ce5644d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 12:09:00 +0100 Subject: [PATCH 70/75] =?UTF-8?q?Style,=20ordre=20des=20ann=C3=A9es=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 4 +- src/pages/observatoire.py | 77 +++++++++++++++++++++++++++------------ src/pages/tableau.py | 4 +- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 42a4def..2efc4a4 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -176,8 +176,8 @@ p.version > a { /* --- Dashboard inputs --- */ .Select--multi .Select-value { - color: var(--primary-color); - background-color: rgba(255, 240, 240, 0.4); + color: var(--primary-color) !important; + background-color: rgba(255, 240, 240, 0.4) !important; } #filters .row > * { diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 2dc7882..e492a0f 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -53,10 +53,13 @@ register_page( image_url=meta_content["image_url"], order=3, ) -options_years = {} +options_years = [] for year in reversed(range(2017, datetime.now().year + 1)): - year = str(year) - options_years[year] = year + option_year = { + "label": str(year), + "value": year, + } + options_years.append(option_year) options_departements = [] for code in departements.keys(): @@ -401,25 +404,39 @@ Alors, on fait comment ? persistence_type="local", ), ), - ), -dbc.Row( - [dbc.Col([ - dcc.Download(id="download-observatoire"), - dbc.Button( - "Prévisualiser les données", - id="btn-observatoire-preview", - className="btn btn-primary", - color="primary", - outline=True, ), - dcc.Input( - id="observatoire-share-url", - readOnly=True, - style={"display": "none"}, - )], lg=12, xl=7)]), -dbc.Row( -dbc.Col( - html.Div(id="observatoire-copy-container"), lg=12, xl=5)) + dbc.Row( + [ + dbc.Col( + [ + dcc.Download( + id="download-observatoire" + ), + dbc.Button( + "Prévisualiser les données", + id="btn-observatoire-preview", + className="btn btn-primary", + color="primary", + outline=True, + ), + dcc.Input( + id="observatoire-share-url", + readOnly=True, + style={"display": "none"}, + ), + ], + lg=12, + xl=7, + ) + ] + ), + dbc.Row( + dbc.Col( + html.Div(id="observatoire-copy-container"), + lg=12, + xl=5, + ) + ), ], ), dbc.Col( @@ -585,8 +602,7 @@ def restore_filters(search, _pathname, stored_filters): Output("observatoire-share-url", "value"), Output("observatoire-copy-container", "children"), *[Input(fp[0], "value") for fp in FILTER_PARAMS], - State("dashboard_url", "href"), - prevent_initial_call=True, + Input("dashboard_url", "href"), ) def sync_observatoire_share_url(*args): # Last arg is href (State), rest are filter values @@ -625,6 +641,7 @@ def sync_observatoire_share_url(*args): children=[ dbc.Button( "Partager cette vue", + id="btn-copy-observatoire", className="btn btn-primary mt-2", title="Copier l'adresse de cette vue filtrée pour la partager.", ) @@ -634,6 +651,20 @@ def sync_observatoire_share_url(*args): return full_url, copy_button +@callback( + Output("observatoire-copy-container", "children", allow_duplicate=True), + Input("btn-copy-observatoire", "n_clicks", allow_optional=True), + prevent_initial_call=True, +) +def show_confirmation(n_clicks): + if n_clicks: + return html.Span( + "Adresse de la vue copiée", + style={"color": "green", "fontWeight": "bold", "marginLeft": "10px"}, + ) + return no_update + + @callback( Output("cards", "children"), Input("dashboard_year", "value"), diff --git a/src/pages/tableau.py b/src/pages/tableau.py index 4ea71c1..8066787 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -28,9 +28,9 @@ from src.utils import ( invert_columns, logger, meta_content, + prepare_table_data, schema, sort_table_data, - prepare_table_data, ) update_date_timestamp = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) @@ -439,7 +439,7 @@ def sync_url_and_reset_button(filter_query, sort_by, hidden_columns, href): className="fa fa-link", children=[ dbc.Button( - "Partager", + "Partager la vue", className="btn btn-primary", title="Copier l'adresse de cette vue (filtres, tris, choix de colonnes) pour la partager.", ) From a54f78875edf9bc080a3f4228e00750ffa91d255 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 21 Mar 2026 12:25:31 +0100 Subject: [PATCH 71/75] =?UTF-8?q?Utilise=20l'ann=C3=A9e=20en=20cours=20pou?= =?UTF-8?q?r=20plafonner=20les=20donn=C3=A9es=20du=20graph=20sources=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 6 +++++- src/pages/observatoire.py | 4 ---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/figures.py b/src/figures.py index 687b5c0..4cbe378 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Literal from urllib.error import HTTPError, URLError @@ -66,6 +67,8 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): "datePublicationDonnees": "publication des données", } + now_year = datetime.now().year + lff = lff.select("uid", type_date, "sourceDataset") lff = lff.unique("uid") @@ -88,7 +91,7 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): lff = lff.with_columns(pl.col(type_date).dt.year().alias("annee")) lff = lff.filter( - pl.col(type_date).is_not_null() & pl.col("annee").is_between(2019, 2025) + pl.col(type_date).is_not_null() & pl.col("annee").is_between(2019, now_year) ) lff = lff.with_columns(pl.col(type_date).cast(pl.String).str.head(7)) lff = ( @@ -98,6 +101,7 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): ) lff = lff.sort(by=["sourceDataset"], descending=False) + dff: pl.DataFrame = lff.collect(engine="streaming") fig = px.bar( diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index e492a0f..b22522c 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -425,16 +425,12 @@ Alors, on fait comment ? style={"display": "none"}, ), ], - lg=12, - xl=7, ) ] ), dbc.Row( dbc.Col( html.Div(id="observatoire-copy-container"), - lg=12, - xl=5, ) ), ], From f4514bf06c1c8b712ba5b5736fe4bd4c74121765 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 23 Mar 2026 07:43:02 +0100 Subject: [PATCH 72/75] =?UTF-8?q?Le=20bouton=20Partager=20n'appara=C3=AEt?= =?UTF-8?q?=20que=20s'il=20y=20a=20des=20filtres?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 63 ++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index b22522c..8a8dee1 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -413,9 +413,9 @@ Alors, on fait comment ? id="download-observatoire" ), dbc.Button( - "Prévisualiser les données", + "Voir les données", id="btn-observatoire-preview", - className="btn btn-primary", + className="btn btn-primary mt-2", color="primary", outline=True, ), @@ -425,14 +425,16 @@ Alors, on fait comment ? style={"display": "none"}, ), ], - ) + lg=12, + xl=6, + ), + dbc.Col( + id="observatoire-copy-container", + lg=12, + xl=6, + ), ] ), - dbc.Row( - dbc.Col( - html.Div(id="observatoire-copy-container"), - ) - ), ], ), dbc.Col( @@ -606,6 +608,7 @@ def sync_observatoire_share_url(*args): href = args[-1] if not href: + print("no update") return no_update, no_update base_url = href.split("?")[0] @@ -622,27 +625,31 @@ def sync_observatoire_share_url(*args): query_string = urllib.parse.urlencode(params) full_url = f"{base_url}?{query_string}" if query_string else base_url + print("query", query_string) - copy_button = dcc.Clipboard( - id="btn-copy-observatoire-url", - target_id="observatoire-share-url", - title="Copier l'URL de cette vue", - style={ - "display": "inline-block", - "fontSize": 20, - "verticalAlign": "top", - "cursor": "pointer", - }, - className="fa fa-link", - children=[ - dbc.Button( - "Partager cette vue", - id="btn-copy-observatoire", - className="btn btn-primary mt-2", - title="Copier l'adresse de cette vue filtrée pour la partager.", - ) - ], - ) + if params: + copy_button = dcc.Clipboard( + id="btn-copy-observatoire-url", + target_id="observatoire-share-url", + title="Copier l'URL de cette vue", + style={ + "display": "inline-block", + "fontSize": 20, + "verticalAlign": "top", + "cursor": "pointer", + }, + className="fa fa-link", + children=[ + dbc.Button( + "Partager cette vue", + id="btn-copy-observatoire", + className="btn btn-primary mt-2", + title="Copier l'adresse de cette vue filtrée pour la partager.", + ) + ], + ) + else: + copy_button = html.Div() return full_url, copy_button From 72da1a15e23d76ceb0e429cde6562ef95fbced83 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 23 Mar 2026 07:20:01 +0100 Subject: [PATCH 73/75] Nettoyage HTML recherche.py --- src/pages/recherche.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pages/recherche.py b/src/pages/recherche.py index b308cf8..910d5d0 100644 --- a/src/pages/recherche.py +++ b/src/pages/recherche.py @@ -78,7 +78,7 @@ layout = html.Div( # className="search_options", # children=[dcc.RadioItems(options=["Acheteur(s)"])], # ), - html.Div(id="search_results"), + dbc.Row(id="search_results"), ], ) @@ -110,8 +110,8 @@ def update_search_results(n_submit, n_clicks, query): # Format output columns, tooltip = setup_table_columns(results, hideable=False) - col_content = ( - html.Div( + col = ( + dbc.Col( children=[ html.H3(f"{org_type.title()}s : {count}"), DataTable( @@ -123,12 +123,13 @@ def update_search_results(n_submit, n_clicks, query): filter_action="none", ), ], + md=6, ) if count > 0 else html.P(f"Aucun {org_type} trouvé.") ) - cols.append(dbc.Col(col_content, width=6)) + cols.append(col) style = {"textAlign": "center", "display": "none"} - return dbc.Row(cols), style + return cols, style return html.P(""), {"textAlign": "center"} From 24bce6e2d6a9c0df96c8e5e30a813ac24cecdaf8 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 23 Mar 2026 07:21:50 +0100 Subject: [PATCH 74/75] Lien vers #sources --- src/pages/a-propos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/a-propos.py b/src/pages/a-propos.py index 80276d0..ea07c8b 100644 --- a/src/pages/a-propos.py +++ b/src/pages/a-propos.py @@ -87,7 +87,7 @@ Vous pouvez consommer les données qui alimentent decp.info dcc.Markdown( """Les données visibles sur ce site proviennent exclusivement de la publication de données ouvertes par les acheteurs publics ou en leur nom, régie par [l'arrêté du 22 décembre 2022](https://www.legifrance.gouv.fr/jorf/id/JORFTEXT000046850496). Leur qualité est donc principalement liée à la qualité de leur saisie par les agents publics, parfois peu aidé·es par la qualité des outils à leur disposition. Je pense que l'analyse de marchés individuels et le comptage de marchés sur des critères autres que financiers sont plutôt fiables. En revanche, certains montants de marché estimés à des valeurs farfelues ([1 euro](https://decp.info/marches/432766947000192025S01301), [1 milliard](https://decp.info/marches/2459004280001320210000000271)) faussent les calculs par aggrégation (sommes, moyennes, médianes) et donc la production de statistiques financières fiables. Acheteurs, acheteuses : s'il vous plaît, essayez d'estimer les montants des marchés publics attribués de manière plus précise. -Quant à l'exhaustivité, je consolide toutes les sources de données exploitables que j'ai pu identifier (voir [statistiques](/statistiques)). Certains profils d'acheteurs ne publient pas leurs données malgré l'obligation réglementaire : +Quant à l'exhaustivité, je consolide toutes les sources de données exploitables que j'ai pu identifier (voir [ci-dessous](/a-propos#sources). Certains profils d'acheteurs ne publient pas leurs données malgré l'obligation réglementaire : - klekoon.fr (ils y travaillent) - safetender.com (Omnikles) From ad3f2cf6540bb77b2fb9d01c2b2131a654aaca42 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 23 Mar 2026 07:48:54 +0100 Subject: [PATCH 75/75] Changelog 2.7.0 --- CHANGELOG.md | 7 +++++++ README.md | 2 +- pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b270d1..4768316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +#### 2.7.0 (23 mars 2026) + +- Remplacement de la page Statistiques par l'observatoire +- Généralisation de la grille dash (`dbc.Row`, `dbc.Col`) +- Ajout de l'histogramme de distances aux pages acheteur et titulaire +- Ajout de la colonne `acheteur_categorie` (commune, État, etc.) + ##### 2.6.2 (22 février 2026) - Correction du téléchargemnent buggé dans /tableau diff --git a/README.md b/README.md index 1d71b7c..6f100eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # decp.info -> v2.6.2 +> v2.7.0 > Outil d'exploration et de téléchargement des données essentielles de la commande publique. => [decp.info](https://decp.info) diff --git a/pyproject.toml b/pyproject.toml index b4c27e3..dc379f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "decp.info" description = "Interface d'exploration et d'analyse des marchés publics français." -version = "2.6.2" +version = "2.7.0" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin@colmo.tech" }