From 472fbb7cbbd261ebe2fd9b9997cb2ac1c1a48b38 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 5 Feb 2026 17:16:31 +0100 Subject: [PATCH 01/52] =?UTF-8?q?Am=C3=A9liorations=20des=20textes=20et=20?= =?UTF-8?q?petits=20ajustements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 12 ++++++++++++ src/figures.py | 2 +- src/pages/tableau.py | 32 +++++++++++++++++++++++++------- src/utils.py | 11 ++++++++++- tests/test_main.py | 3 --- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index ccb63d0..2fd4da3 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -301,6 +301,15 @@ table.cell-table th { vertical-align: center; } +.dash-table-container + .dash-spreadsheet-container + .dash-spreadsheet-inner + .cell-table + .dash-filter + input[type="text"]::placeholder { + color: #999; +} + .dash-filter--case { display: none; } @@ -375,6 +384,9 @@ button.show-hide { margin-right: 10px; } */ +#btn-copy-url:before { +} + /* Dropdowns */ .Select-placeholder { color: #333 !important; diff --git a/src/figures.py b/src/figures.py index fb07e7d..e95b779 100644 --- a/src/figures.py +++ b/src/figures.py @@ -326,7 +326,7 @@ class DataTable(dash_table.DataTable): page_action=page_action, filter_options={ "case": "insensitive", - "placeholder_text": "", + "placeholder_text": "Filtre de colonne...", }, sort_action=sort_action, sort_mode="multi", diff --git a/src/pages/tableau.py b/src/pages/tableau.py index 7b9f913..920fc20 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -130,7 +130,7 @@ layout = [ ], ), dcc.Markdown( - f"Ce tableau vous permet d'appliquer un filtre sur une ou plusieurs colonnes, et ainsi produire la liste de marchés dont vous avez besoin ([exemple de filtre](/tableau?filtres=%7Bacheteur_id%7D+icontains+24350013900189+%26%26+%7BdateNotification%7D+icontains+2025%2A+%26%26+%7Bmontant%7D+i%3C+40000+%26%26+%7Bobjet%7D+icontains+voirie&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2Cacheteur_departement_code%2CsourceDataset)). Par défaut seules quelques colonnes sont affichées, mais vous pouvez en afficher jusqu'à {str(df.width)} en cliquant sur le bouton **Colonnes affichées**. Cet outil est assez puissant, je vous recommande de lire le mode d'emploi pour en tirer pleinement partie.", + f"Ce tableau contient tous les marchés attribués en France. Il vous permet d'appliquer un filtre sur une ou plusieurs colonnes, et ainsi produire la liste de marchés dont vous avez besoin (exemples : [marchés de voirie < 40 k€ en 2025](/tableau?filtres=%7Bacheteur_id%7D+icontains+24350013900189+%26%26+%7BdateNotification%7D+icontains+2025%2A+%26%26+%7Bmontant%7D+i%3C+40000+%26%26+%7Bobjet%7D+icontains+voirie&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2Cacheteur_departement_code%2CsourceDataset), [marchés > 500 k€ avec clause sociale attribués à des PME à plus de 100 km dans le Var](/tableau?filtres=%7Btitulaire_categorie%7D+icontains+PME+%26%26+%7Btitulaire_distance%7D+i%3E+100+%26%26+%7Bmontant%7D+i%3E+500000+%26%26+%7Bacheteur_departement_code%7D+icontains+83+%26%26+%7BconsiderationsSociales%7D+icontains+clause&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2CconsiderationsSociales%2Ctitulaire_distance%2Cacheteur_departement_code%2Ctitulaire_categorie%2CsourceDataset)). Par défaut seules quelques colonnes sont affichées, mais vous pouvez en afficher jusqu'à {str(df.width)} en cliquant sur le bouton **Choisir les colonnes**. Cet outil est assez puissant, je vous recommande de lire le mode d'emploi pour en tirer pleinement partie.", style={"maxWidth": "1000px"}, ), html.Div( @@ -148,7 +148,7 @@ layout = [ dbc.Button("Mode d'emploi", id="tableau_help_open"), dbc.Modal( [ - dbc.ModalHeader(dbc.ModalTitle("Header")), + dbc.ModalHeader(dbc.ModalTitle("Mode d'emploi")), dbc.ModalBody( dcc.Markdown( dangerously_allow_html=True, @@ -157,6 +157,10 @@ layout = [ Pour voir la définition d'une colonne, passez votre souris sur son en-tête. + ##### Vos réglages sont persistents + + Les filtres, les tris et le choix de colonnes sont automatiquement enregistrés dans votre navigateur et persistent même si vous changez de page ou si vous fermez votre navigateur. À votre retour, vous retrouverez cette page comme vous l'avez laissée. + ##### Appliquer des filtres Vous pouvez appliquer un filtre pour chaque colonne en entrant du texte sous le nom de la colonne, puis en tapant sur `Entrée`. @@ -174,7 +178,7 @@ layout = [ - pour chercher du texte qui **commence par** votre texte, entrez `texte*`. C'est par exemple utile pour filtrer des acheteurs ou titulaires par numéro SIREN (`123456789*`) ou les marchés sur une année en particulier (`2024*`) - pour chercher du texte qui **finit par** votre texte, entrez `*texte` - Vous pouvez filtrer plusieurs colonnes à la fois. Vos filtres sont remis à zéro quand vous rafraîchissez la page. + Vous pouvez filtrer plusieurs colonnes à la fois. ##### Trier les données @@ -188,11 +192,11 @@ layout = [ Par défaut, un nombre réduit de colonnes est affiché pour ne pas surcharger la page. Mais vous avez le choix parmi {str(df.width)} colonnes, ce serait dommage de vous limiter ! - Pour afficher plus de colonnes, cliquez sur le bouton **Colonnes affichées** et cochez les colonnes pour les afficher. + Pour afficher plus de colonnes, cliquez sur le bouton **Choisir les colonnes** et cochez les colonnes pour les afficher. ##### Partager une vue - Une vue est un ensemble de filtres, de tris et de choix de colonnes que vous avez appliqués. Cliquez sur l'icône drawing pour copier une adresse Web qui reproduit la vue courante à l'identique : en la collant dans la barre d'adresse d'un navigateur, vous ouvrez la vue Tableau avec les mêmes paramètres. + Une vue est un ensemble de filtres, de tris et de choix de colonnes que vous avez appliqués. Cliquez sur **Partager** pour copier une adresse Web qui reproduit la vue courante à l'identique : en la collant dans la barre d'adresse d'un navigateur, vous ouvrez la vue Tableau avec les mêmes paramètres. Pratique pour partager une vue avec un·e collègue, sur les réseaux sociaux, ou la sauvegarder pour plus tard. @@ -225,9 +229,10 @@ layout = [ ), # Bouton modal des colonnes affichées dbc.Button( - "Colonnes affichées", + "Choisir les colonnes", id="tableau_columns_open", className="column_list", + title="Choisir les colonnes à afficher et masquer", ), html.P("lignes", id="nb_rows"), html.Div(id="copy-container"), @@ -241,7 +246,7 @@ layout = [ dcc.Store(id="filtered_data", storage_type="memory"), html.P("Données mises à jour le " + str(update_date)), dbc.Button( - "Remise à zéro", + "Remettre à zéro", title="Supprime tous les filtres et les tris. Autrement ils sont conservés même si vous fermez la page.", id="btn-tableau-reset", ), @@ -434,6 +439,13 @@ def sync_url_and_reset_button(filter_query, sort_by, hidden_columns, href): "cursor": "pointer", }, className="fa fa-link", + children=[ + dbc.Button( + "Partager", + className="btn btn-primary", + title="Copier l'adresse de cette vue (filtres, tris, choix de colonnes) pour la partager.", + ) + ], ) return full_url, copy_button @@ -527,3 +539,9 @@ def reset_view(n_clicks): @callback(Input("tableau_url", "pathname")) def cb_add_canonical_link(pathname): add_canonical_link(pathname) + + +# @callback(Input("tableau_url", "pathname"), Output("btn-copy-url", "children")) +# def cb_add_canonical_link(pathname): +# add_canonical_link(pathname) +# diff --git a/src/utils.py b/src/utils.py index c4c2c98..ccd50ef 100644 --- a/src/utils.py +++ b/src/utils.py @@ -242,6 +242,15 @@ def get_decp_data() -> pl.DataFrame: # Convertir les colonnes booléennes en chaînes de caractères lff = booleans_to_strings(lff) + # Mention pour les org dont on a pas le nom + for col in ["acheteur_nom", "titulaire_nom"]: + lff = lff.with_columns( + pl.when(pl.col(col).is_null()) + .then(pl.lit("[Identifiant non reconnu dans la base INSEE]")) + .otherwise(pl.col(col)) + .name.keep() + ) + # Bizarrement je ne peux pas faire lff = lff.fill_null("") ici # ça génère une erreur dans la page acheteur (acheteur_data.table) : # AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import) @@ -673,7 +682,7 @@ def get_button_properties(height): else: download_disabled = False download_text = "Télécharger au format Excel" - download_title = "" + download_title = "Télécharger les données telles qu'affichées au format Excel" return download_disabled, download_text, download_title diff --git a/tests/test_main.py b/tests/test_main.py index 3d1920a..453a806 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,5 +1,3 @@ -from time import sleep - from dash.testing.composite import DashComposite from selenium.webdriver import Keys from selenium.webdriver.common.by import By @@ -58,6 +56,5 @@ def test_002_filter_persistence(dash_duo: DashComposite): filter_input = open_page_and_check_filter_input() filter_input.send_keys("11") # a UID that doesn't exist filter_input.send_keys(Keys.ENTER) - sleep(1) filter_input = open_page_and_check_filter_input() assert filter_input.get_attribute("value") == "11" From b303a8bea67fe9de8adab325e896cd1a8a89eda5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 5 Feb 2026 17:29:54 +0100 Subject: [PATCH 02/52] Bump version 2.6.0 et changelog --- CHANGELOG.md | 8 ++++++++ README.md | 2 +- pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52515f9..b82194d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +#### 2.6.0 (5 février 2026) + +- Suite de la refonte graphique +- Persistence des filtres, des tris et des choix de colonnes sur toutes les pages +- Joli tableau pour choisir les colonnes à afficher +- Meilleure gestion des acheteurs et titulaires absents de la base SIRENE +- Amélioration du SEO (liens canoniques) + ##### 2.5.1 (29 janvier 2026) - Mise en production un peu hâtive ([#67](https://github.com/ColinMaudry/decp.info/issues/67), [#68](https://github.com/ColinMaudry/decp.info/issues/68)) diff --git a/README.md b/README.md index f710191..ecfa81b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # decp.info -> v2.5.1 +> v2.6.0 > Outil d'exploration et de téléchargement des données essentielles de la commande publique. => [decp.info](https://decp.info) diff --git a/pyproject.toml b/pyproject.toml index 261b9bb..35749a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "decp.info" description = "Interface d'exploration et d'analyse des marchés publics français." -version = "2.5.1" +version = "2.6.0" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin@colmo.tech" } From eb5be1972e7a02a7368ebbf2834ad45ea4ac8f7f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 5 Feb 2026 18:10:20 +0100 Subject: [PATCH 03/52] Correction de soucis de sauvegarde des colonnes dans acheteur.py --- src/pages/acheteur.py | 3 +-- src/pages/titulaire.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 1a3e915..90e0fe6 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -318,7 +318,7 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> tuple: Output("btn-download-filtered-data-acheteur", "disabled"), Output("btn-download-filtered-data-acheteur", "children"), Output("btn-download-filtered-data-acheteur", "title"), - Output("filter-cleanup-trigger-acheteur", "data", allow_duplicate=True), + Output("filter-cleanup-trigger-acheteur", "data"), Input("acheteur_url", "href"), Input("acheteur_data", "data"), Input("acheteur_datatable", "page_current"), @@ -326,7 +326,6 @@ def get_acheteur_marches_data(url, acheteur_year: str) -> tuple: Input("acheteur_datatable", "filter_query"), Input("acheteur_datatable", "sort_by"), State("acheteur_datatable", "data_timestamp"), - prevent_initial_call=True, ) def get_last_marches_data( href, data, page_current, page_size, filter_query, sort_by, data_timestamp diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 51937c4..23c3cf6 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -326,7 +326,7 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: Output("btn-download-filtered-data-titulaire", "disabled"), Output("btn-download-filtered-data-titulaire", "children"), Output("btn-download-filtered-data-titulaire", "title"), - Output("filter-cleanup-trigger-titulaire", "data", allow_duplicate=True), + Output("filter-cleanup-trigger-titulaire", "data"), Input(component_id="titulaire_url", component_property="href"), Input("titulaire_data", "data"), Input("titulaire_datatable", "page_current"), @@ -334,7 +334,6 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple: Input("titulaire_datatable", "filter_query"), Input("titulaire_datatable", "sort_by"), State("titulaire_datatable", "data_timestamp"), - config_prevent_initial_callbacks=True, ) def get_last_marches_data( href, data, page_current, page_size, filter_query, sort_by, data_timestamp From af89bb0630cf5842ec4a13f1a7bb8020d7f9bf6f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Thu, 5 Feb 2026 18:22:17 +0100 Subject: [PATCH 04/52] =?UTF-8?q?Plus=20grosses=20cases=20=C3=A0=20cocher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 2fd4da3..e76e1f2 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -392,6 +392,13 @@ button.show-hide { color: #333 !important; } +/* Checkboxes */ + +input[type="checkbox"] { + height: 17px; + width: 17px; +} + /* Tooltips */ .dash-tooltip, .dash-table-tooltip { From 5c43bfea78dacf7a4e69f6493b14f61dade39cb0 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 15 Feb 2026 16:50:09 +0100 Subject: [PATCH 05/52] =?UTF-8?q?AM=C3=A9lioration=20de=20la=20g=C3=A9n?= =?UTF-8?q?=C3=A9ration=20de=20liens=20canoniques=20(pas=20de=20doublons)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app.py | 1 + src/utils.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/app.py b/src/app.py index 3f45aed..eb1931c 100644 --- a/src/app.py +++ b/src/app.py @@ -87,6 +87,7 @@ app.index_string = """ {%title%} {%favicon%} {%css%} + {%app_entry%} diff --git a/src/utils.py b/src/utils.py index ccd50ef..c8c41b8 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,6 +1,7 @@ import json import logging import os +import re import uuid from collections import OrderedDict from time import localtime, sleep @@ -443,7 +444,7 @@ def get_default_hidden_columns(page): "codeCPV", "dureeRestanteMois", ] - elif page == "titulaire": + elif page == "tableau": displayed_columns = os.getenv("DISPLAYED_COLUMNS") else: displayed_columns = os.getenv("DISPLAYED_COLUMNS") @@ -742,10 +743,16 @@ def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dic def add_canonical_link(pathname): @dash.hooks.index() - def update_index(html_string): + def update_index(html_string: str): url = f"https://{domain_name}{pathname}" canonical_tag = f'' - return html_string.replace("", f"{canonical_tag}\n ") + html_string = re.sub( + r'', + canonical_tag, + html_string, + ) + html_string = html_string.replace("", canonical_tag) + return html_string df: pl.DataFrame = get_decp_data() From 2b0b048520ee243486ceb5a37abe5b660266e086 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 15 Feb 2026 16:54:30 +0100 Subject: [PATCH 06/52] =?UTF-8?q?Am=C3=A9lioration=20du=20nom=20titulaire/?= =?UTF-8?q?acheteur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 9 +++++---- src/pages/titulaire.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 90e0fe6..53742d0 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -34,12 +34,12 @@ from src.utils import ( def get_title(acheteur_id: str = None) -> str: - df_acheteur = df_acheteurs.filter(pl.col("acheteur_id") == acheteur_id).select( + acheteur_nom = df_acheteurs.filter(pl.col("acheteur_id") == acheteur_id).select( "acheteur_nom" ) - acheteur_nom = df_acheteur.item(0, 0) - - return f"Marchés publics attribués par {acheteur_nom} | decp.info" + if acheteur_nom.height > 0: + return f"Marchés publics attribués par {acheteur_nom.item(0, 0)} | decp.info" + return "Marchés publics attribués | decp.info" register_page( @@ -480,4 +480,5 @@ def reset_view(n_clicks): @callback(Input("acheteur_url", "pathname")) def cb_add_canonical_link(pathname): + print("coucou 2") add_canonical_link(pathname) diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 23c3cf6..59ab2bc 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -34,12 +34,12 @@ from src.utils import ( def get_title(titulaire_id: str = None) -> str: - titulaire_nom = ( - df_titulaires.filter(pl.col("titulaire_id") == titulaire_id) - .select("titulaire_nom") - .item(0, 0) + titulaire_nom = df_titulaires.filter(pl.col("titulaire_id") == titulaire_id).select( + "titulaire_nom" ) - return f"Marchés publics remportés par {titulaire_nom} | decp.info" + if titulaire_nom.height > 0: + return f"Marchés publics remportés par {titulaire_nom.item(0, 0)} | decp.info" + return "Marchés publics remportés | decp.info" register_page( From adda58badaf734d590659ad300dd9c4fbe866167 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Feb 2026 14:24:46 +0100 Subject: [PATCH 07/52] =?UTF-8?q?Suppresion=20des=20liens=20canoniques,=20?= =?UTF-8?q?pas=20trouv=C3=A9=20comment=20les=20ins=C3=A9rer=20assez=20t?= =?UTF-8?q?=C3=B4t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 7 ------- src/pages/tableau.py | 12 ------------ src/pages/titulaire.py | 6 ------ src/utils.py | 16 ---------------- 4 files changed, 41 deletions(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 53742d0..9e083c0 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -17,7 +17,6 @@ from dash import ( from src.callbacks import get_top_org_table from src.figures import DataTable, make_column_picker, point_on_map from src.utils import ( - add_canonical_link, columns, df, df_acheteurs, @@ -476,9 +475,3 @@ def toggle_acheteur_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] - - -@callback(Input("acheteur_url", "pathname")) -def cb_add_canonical_link(pathname): - print("coucou 2") - add_canonical_link(pathname) diff --git a/src/pages/tableau.py b/src/pages/tableau.py index 920fc20..2bf295d 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -22,7 +22,6 @@ from dash import ( from figures import make_column_picker from src.figures import DataTable from src.utils import ( - add_canonical_link, columns, df, filter_table_data, @@ -534,14 +533,3 @@ def toggle_tableau_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] - - -@callback(Input("tableau_url", "pathname")) -def cb_add_canonical_link(pathname): - add_canonical_link(pathname) - - -# @callback(Input("tableau_url", "pathname"), Output("btn-copy-url", "children")) -# def cb_add_canonical_link(pathname): -# add_canonical_link(pathname) -# diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 59ab2bc..92ff313 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -17,7 +17,6 @@ from dash import ( from src.callbacks import get_top_org_table from src.figures import DataTable, make_column_picker, point_on_map from src.utils import ( - add_canonical_link, columns, df, df_titulaires, @@ -490,8 +489,3 @@ def toggle_titulaire_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] - - -@callback(Input("titulaire_url", "pathname")) -def cb_add_canonical_link(pathname): - add_canonical_link(pathname) diff --git a/src/utils.py b/src/utils.py index c8c41b8..8c9dae7 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,12 +1,10 @@ import json import logging import os -import re import uuid from collections import OrderedDict from time import localtime, sleep -import dash import polars as pl import polars.selectors as cs from dash import no_update @@ -741,20 +739,6 @@ def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dic return jsonld -def add_canonical_link(pathname): - @dash.hooks.index() - def update_index(html_string: str): - url = f"https://{domain_name}{pathname}" - canonical_tag = f'' - html_string = re.sub( - r'', - canonical_tag, - html_string, - ) - html_string = html_string.replace("", canonical_tag) - return html_string - - df: pl.DataFrame = get_decp_data() schema = df.collect_schema() From 367e5f64ff599d8fe47cee8f6584acaa5b1eaabe Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Feb 2026 14:50:29 +0100 Subject: [PATCH 08/52] Changelog 2.6.1 --- CHANGELOG.md | 4 ++++ README.md | 2 +- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b82194d..09c3d2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +#### 2.6.1 (17 février 2026) + +- Corrections la création des liens canoniques (SEO) + #### 2.6.0 (5 février 2026) - Suite de la refonte graphique diff --git a/README.md b/README.md index ecfa81b..530fdca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # decp.info -> v2.6.0 +> v2.6.1 > Outil d'exploration et de téléchargement des données essentielles de la commande publique. => [decp.info](https://decp.info) diff --git a/pyproject.toml b/pyproject.toml index 35749a7..96a606a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "decp.info" description = "Interface d'exploration et d'analyse des marchés publics français." -version = "2.6.0" +version = "2.6.1" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin@colmo.tech" } From 4619dd2708b25ce62276192078772f9a793934d5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 22 Feb 2026 18:39:31 +0100 Subject: [PATCH 09/52] =?UTF-8?q?Correction=20du=20t=C3=A9l=C3=A9chargemne?= =?UTF-8?q?nt=20bugg=C3=A9=20dans=20/tableau=20+=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 +++- src/pages/acheteur.py | 3 ++- src/pages/tableau.py | 2 +- src/pages/titulaire.py | 3 ++- tests/conftest.py | 15 ++++++++++++++- tests/test_main.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 96a606a..342ff74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dev = [ "selenium", "webdriver-manager", "dash[testing]", + "fastexcel" ] [tool.pytest.ini_options] @@ -38,6 +39,7 @@ testpaths = [ "tests" ] env = [ - "DATA_FILE_PARQUET_PATH=tests/test.parquet" + "DATA_FILE_PARQUET_PATH=tests/test.parquet", + "DEVELOPMENT=true" ] addopts = "-p no:warnings" diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 9e083c0..199c757 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -1,4 +1,5 @@ import datetime +from typing import Any import dash_bootstrap_components as dbc import polars as pl @@ -352,7 +353,7 @@ def get_top_titulaires(data): ) def download_acheteur_data( n_clicks, - data: [dict], + data: list[dict[str, Any]], acheteur_nom: str, annee: str, ): diff --git a/src/pages/tableau.py b/src/pages/tableau.py index 2bf295d..56fb9b8 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -326,7 +326,7 @@ def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None): if filter_query: lff = filter_table_data(lff, filter_query, "tab download") - if len(sort_by) > 0: + if sort_by and len(sort_by) > 0: lff = sort_table_data(lff, sort_by) def to_bytes(buffer): diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index 92ff313..b7e9001 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -1,4 +1,5 @@ import datetime +from typing import Any import dash_bootstrap_components as dbc import polars as pl @@ -366,7 +367,7 @@ def get_top_acheteurs(data): ) def download_titulaire_data( n_clicks, - data: [dict], + data: list[dict[str, Any]], titulaire_nom: str, annee: str, ): diff --git a/tests/conftest.py b/tests/conftest.py index 7531524..b14e0f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,8 +43,21 @@ def test_data(): pl.DataFrame(data).write_parquet("tests/test.parquet") yield path + if os.path.exists(path): + os.unlink(path) + print(path, "deleted") + def pytest_setup_options(): options = Options() - options.add_argument("--window-size=1200,800") + options.add_argument("--window-size=1200,1200 ") + options.add_experimental_option( + "prefs", + { + "download.default_directory": "/home/colin/git/decp.info", + "download.prompt_for_download": False, + "download.directory_upgrade": True, + "safebrowsing.enabled": True, + }, + ) return options diff --git a/tests/test_main.py b/tests/test_main.py index 453a806..1b8e7db 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,3 +1,4 @@ +import polars as pl from dash.testing.composite import DashComposite from selenium.webdriver import Keys from selenium.webdriver.common.by import By @@ -58,3 +59,31 @@ def test_002_filter_persistence(dash_duo: DashComposite): filter_input.send_keys(Keys.ENTER) filter_input = open_page_and_check_filter_input() assert filter_input.get_attribute("value") == "11" + + +def test_003_tableau_download(dash_duo: DashComposite): + from pages.acheteur import download_acheteur_data + from pages.tableau import download_data + from pages.titulaire import download_titulaire_data + from src.app import app + + # Juste pour instancier l'app + print(app.server.name) + + dicts = pl.read_parquet("tests/test.parquet").to_dicts() + + outputs = [ + download_data(1, "", [], None), + download_acheteur_data(1, dicts, "a1", "2025"), + download_titulaire_data(1, dicts, "t1", "2025"), + ] + for output in outputs: + assert isinstance(output, dict) + for f in ["content", "filename", "type", "base64"]: + assert f in output + assert isinstance(output["content"], str) and len(output["content"]) > 100 + assert isinstance(output["filename"], str) and output["filename"].startswith( + "decp_" + ) + assert output["type"] is None + assert output["base64"] is True From 0db800fdabbb7e3f301a9e883191a279d68d958d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 22 Feb 2026 18:41:01 +0100 Subject: [PATCH 10/52] Changelog 2.6.2 --- CHANGELOG.md | 6 +++++- README.md | 2 +- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c3d2a..7b270d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -#### 2.6.1 (17 février 2026) +##### 2.6.2 (22 février 2026) + +- Correction du téléchargemnent buggé dans /tableau + +##### 2.6.1 (17 février 2026) - Corrections la création des liens canoniques (SEO) diff --git a/README.md b/README.md index 530fdca..1d71b7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # decp.info -> v2.6.1 +> v2.6.2 > Outil d'exploration et de téléchargement des données essentielles de la commande publique. => [decp.info](https://decp.info) diff --git a/pyproject.toml b/pyproject.toml index 342ff74..cdca66b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "decp.info" description = "Interface d'exploration et d'analyse des marchés publics français." -version = "2.6.1" +version = "2.6.2" requires-python = ">= 3.10" authors = [ { name = "Colin Maudry", email = "colin@colmo.tech" } From c7d1a5ec731847f0aecd0aa2331d819e5d25958d Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 13 Mar 2026 19:22:48 +0100 Subject: [PATCH 11/52] =?UTF-8?q?D=C3=A9but=20de=20dashboard=20avec=20quel?= =?UTF-8?q?ques=20filtres=20et=20viz=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 4 + src/figures.py | 132 +++++++++++++++++++++++-- src/pages/statistiques.py | 201 ++++++++++++++++++++++++++++---------- src/utils.py | 10 ++ 4 files changed, 290 insertions(+), 57 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index e76e1f2..a183642 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -151,6 +151,10 @@ p.version > a { max-width: 900px; } +.seeBorder { + border: dotted 1px green; +} + /* --- Search Page --- */ .tagline { text-align: center; diff --git a/src/figures.py b/src/figures.py index e95b779..f06ae37 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,6 +1,8 @@ import json from typing import Literal +from urllib.error import HTTPError, URLError +import dash_bootstrap_components as dbc import plotly.express as px import plotly.graph_objects as go import polars as pl @@ -161,8 +163,11 @@ def get_barchart_sources(df_source: pl.DataFrame, type_date: str): def get_sources_tables(source_path) -> html.Div: - df = pl.read_csv(source_path) - df = df.with_columns( + try: + dff = pl.read_csv(source_path) + except (URLError, HTTPError): + return html.Div("Erreur de connexion") + dff = dff.with_columns( ( pl.lit(' html.Div: + pl.lit("") ).alias("nom") ) - df = df.drop("url", "unique") - df = df.sort(by=["nb_marchés"], descending=True) + dff = dff.drop("url", "unique") + dff = dff.sort(by=["nb_marchés"], descending=True) columns = { "nom": "Nom de la source", @@ -184,7 +189,7 @@ def get_sources_tables(source_path) -> html.Div: datatable = dash_table.DataTable( id="source_table", - data=df.to_dicts(), + data=dff.to_dicts(), columns=[ { "name": columns[i], @@ -193,7 +198,7 @@ def get_sources_tables(source_path) -> html.Div: "type": "text", "format": {"nully": "N/A"}, } - for i in df.schema.names() + for i in dff.schema.names() ], style_cell_conditional=[ { @@ -416,6 +421,121 @@ def get_duplicate_matrix() -> html.Div: ) +def get_geographic_maps(dff: pl.DataFrame) -> list | None: + """ + Génère les cartes géographiques pour la métropole et les DOM-TOM. + """ + + # Seulement si les données ne sont pas trop importantes + + if dff.height > 10000: + return [] + + # Liste des codes départements Outre-Mer + dom_codes = ["971", "972", "973", "974", "976"] + + # Couleurs accessibles (Okabe-Ito) + color_acheteur = "#E69F00" # Orange + color_titulaire = "#56B4E9" # Bleu ciel + + regions = { + "Métropole": dff.filter(~pl.col("acheteur_departement_code").is_in(dom_codes)) + } + + # Ajout des DOM s'ils ont des données + for code in dom_codes: + dom_data = dff.filter(pl.col("acheteur_departement_code") == code) + if dom_data.height > 0: + name = f"Département {code}" + if code == "971": + name = "Guadeloupe" + elif code == "972": + name = "Martinique" + elif code == "973": + name = "Guyane" + elif code == "974": + name = "La Réunion" + elif code == "976": + name = "Mayotte" + regions[name] = dom_data + + cols = [] + for name, region_df in regions.items(): + fig = go.Figure() + + # Trace Acheteurs + mask_acheteur = region_df.filter( + pl.col("acheteur_latitude").is_not_null() + & pl.col("acheteur_longitude").is_not_null() + ) + if mask_acheteur.height > 0: + fig.add_trace( + go.Scattergeo( + lat=mask_acheteur["acheteur_latitude"], + lon=mask_acheteur["acheteur_longitude"], + mode="markers", + marker=dict(size=6, color=color_acheteur, opacity=0.5), + name="Acheteurs", + text=mask_acheteur["acheteur_nom"], + ) + ) + + # Trace Titulaires + mask_titulaire = region_df.filter( + pl.col("titulaire_latitude").is_not_null() + & pl.col("titulaire_longitude").is_not_null() + ) + if mask_titulaire.height > 0: + fig.add_trace( + go.Scattergeo( + lat=mask_titulaire["titulaire_latitude"], + lon=mask_titulaire["titulaire_longitude"], + mode="markers", + marker=dict(size=6, color=color_titulaire, opacity=0.5), + name="Titulaires", + text=mask_titulaire["titulaire_nom"], + ) + ) + + # Configuration spécifique de la vue + geo_config = dict( + projection_type="mercator", + showland=True, + landcolor="lightgray", + showcountries=True, + countrycolor="white", + fitbounds="locations" if name != "Métropole" else False, + resolution=50, + ) + + if name == "Métropole": + geo_config["lataxis_range"] = [41, 52] + geo_config["lonaxis_range"] = [-5, 10] + + fig.update_layout( + title=name, + geo=geo_config, + margin=dict(l=0, r=0, t=30, b=0), + height=400 if name == "Métropole" else 300, + showlegend=(name == "Métropole"), + legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), + ) + + # Taille de la colonne : 4 slots (md=12) pour Métropole, 1 slot (md=3) pour DOM + # On suppose une grille de 12 colonnes où 1 card = md=3 + col_width = 12 if name == "Métropole" else 3 + cols.append( + dbc.Col( + dcc.Graph(figure=fig, config={"displayModeBar": False}), + width=12, + md=col_width, + className="mb-4", + ) + ) + + return cols + + def make_column_picker(page: str): table_data = [] table_columns = [ diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 058f660..1876dfa 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -1,14 +1,20 @@ -from datetime import datetime +from datetime import datetime, timedelta -from dash import dcc, html, register_page +import dash_bootstrap_components as dbc +import polars as pl +import polars.selectors as cs +from dash import Input, Output, callback, dcc, html, register_page from src.figures import ( - get_barchart_sources, - get_duplicate_matrix, - get_map_count_marches, - get_yearly_statistics, + get_geographic_maps, +) +from src.utils import ( + departements, + df, + format_number, + get_enum_values_as_dict, + meta_content, ) -from src.utils import df, format_number, get_statistics, meta_content name = "Statistiques" @@ -21,13 +27,21 @@ register_page( image_url=meta_content["image_url"], order=3, ) +options_years = {} +for year in reversed(range(2017, datetime.now().year + 1)): + year = str(year) + options_years[year] = year + +options_departements = {} +for code, obj in departements.items(): + options_departements[code] = f"{obj['departement']} ({code})" -statistics: dict = get_statistics() -today_str = datetime.fromisoformat(statistics["datetime"]).strftime("%d/%m/%Y") layout = [ + dcc.Store(id="dashboard-filters"), + dcc.Location(id="dashboard_url"), html.Div( - className="container", + className="container-fluid", children=[ html.H2(name), dcc.Loading( @@ -35,51 +49,136 @@ layout = [ id="loading-statistques", type="default", children=[ - html.Div( - children=[ - dcc.Markdown(f""" - La publication de données essentielles de marchés publics (DECP) est souvent effectuée par - les plateformes de marchés publics (profils d'acheteurs). Cependant, certaines plateformes ne publient pas, - ou publient d'une manière qui rend la récupération des données compliquée. Les données présentées sur ce site - ne représentent donc pas tous les marchés attribués en France, seulement une partie significative. - - L'ajout de nouvelles plateformes [est en cours](https://github.com/ColinMaudry/decp-processing/issues?q=is%3Aissue%20label%3A%22source%20de%20donn%C3%A9es%22), - toutes les [contributions](/a-propos#contribuer) sont les bienvenues pour atteindre l'exhaustivité. - - Les statistiques publiées sur cette page ont été produites automatiquement à partir des données les plus récentes ({today_str}). - """), - html.H3( - "Statistiques générales sur les marchés", - id="marches", + dbc.Row( + [ + dbc.Col( + width=12, + md=3, + id="filters", + children=[ + html.H5("Période d'attribution"), + dbc.Row( + dcc.Dropdown( + id="dashboard_year", + options=options_years, + placeholder="12 derniers mois", + ), + ), + html.H5("Acheteur"), + dbc.Row( + dcc.Dropdown( + id="dashboard_acheteur_categorie", + options=get_enum_values_as_dict( + "acheteur_categorie" + ), + placeholder="Catégorie d'acheteur", + ), + ), + dbc.Row( + dcc.Dropdown( + id="dashboard_acheteur_departement_code", + searchable=True, + multi=True, + placeholder="Code département acheteur", + options=options_departements, + ), + ), + ], ), - html.P( - "À noter qu'une fois un marché attribué ses données essentielles peuvent malheureusement mettre plusieurs mois à être publiées par l'acheteur." + dbc.Col( + width=12, + md=9, + id="cards", + children=[ + dbc.Row( + ( + dbc.Col( + width=6, + md=4, + className="card", + id="card_basic_counts", + ) + ), + className="mb-4", + ), + dbc.Row(id="maps_row"), + ], ), - html.H4("Statistiques cumulées"), - dcc.Markdown(f""" - - Nombre de marchés publics et accord-cadres : {format_number(statistics["nb_marches"])} - - Nombre d'acheteurs publics (SIRET) : {format_number(statistics["nb_acheteurs_uniques"])} - - Nombre de titulaires (SIRET) : {format_number(statistics["nb_titulaires_uniques"])} - - Je ne publie pas encore de statistiques sur les montants de marchés car je n'ai pas encore trouvé la bonne formule pour traiter les trop nombreux montants fantaisistes qui polluent les calculs. - """), - html.H4("Statistiques par année"), - get_yearly_statistics(statistics, today_str), - dcc.Graph(figure=get_map_count_marches()), - get_duplicate_matrix(), - html.H3("Nombre de marchés par source dans le temps"), - dcc.Graph( - figure=get_barchart_sources(df, "dateNotification") - ), - dcc.Graph( - figure=get_barchart_sources( - df, "datePublicationDonnees" - ) - ), - ], + ] ) ], ), ], - ) + ), ] + + +@callback( + Output("card_basic_counts", "children"), + Output("maps_row", "children"), + Input("dashboard_year", "value"), + Input("dashboard_acheteur_categorie", "value"), + Input("dashboard_acheteur_departement_code", "value"), +) +def udpate_dashboard_cards( + dashboard_year, dashboard_acheteur_categorie, dashboard_acheteur_departement_code +): + lff: pl.LazyFrame = df.lazy() + lff = lff.select( + "uid", + cs.starts_with("acheteur"), + cs.starts_with("titulaire"), + "dateNotification", + "montant", + ) + + # Application des filtres + + if dashboard_year: + lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) + else: + lff = lff.filter( + pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) + ) + + if dashboard_acheteur_categorie: + lff = lff.filter( + pl.col("acheteur_categorie").is_in(dashboard_acheteur_categorie) + ) + + if ( + dashboard_acheteur_departement_code + and len(dashboard_acheteur_departement_code) >= 2 + ): + lff = lff.filter( + pl.col("acheteur_departement_code").is_in( + dashboard_acheteur_departement_code + ) + ) + + # Génération des métriques + dff = lff.collect() + + nb_acheteurs = dff.select("acheteur_id").n_unique() + nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() + + df_per_uid = ( + dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first()) + ) + total_montant = df_per_uid.select(pl.col("montant").sum()).item() + nb_marches = df_per_uid.height + + card_basic_counts = [ + html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), + html.P( + ["Nombre d'acheteurs : ", html.Strong(str(format_number(nb_acheteurs)))] + ), + html.P( + ["Nombre de titulaires : ", html.Strong(str(format_number(nb_titulaires)))] + ), + html.P(["Montant total : ", html.Strong(format_number(total_montant) + " €")]), + ] + + geographic_maps = get_geographic_maps(dff) + + return card_basic_counts, geographic_maps diff --git a/src/utils.py b/src/utils.py index 8c9dae7..992a99b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -685,6 +685,16 @@ def get_button_properties(height): return download_disabled, download_text, download_title +def get_enum_values_as_dict(column_name): + for column in data_schema: + if column == column_name: + options = {} + for value in data_schema[column]["enum"]: + options[value] = value + return options + return {"not_found": "not found"} + + def invert_columns(columns): """ Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa. From d9f97cf8b3c02e6b15e8b08766e49b86ab6a1a30 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Fri, 13 Mar 2026 23:59:54 +0100 Subject: [PATCH 12/52] =?UTF-8?q?Modification=20de=20map=5Fcount=5Fmarches?= =?UTF-8?q?=20(plus=20efficace,=20utilise=20le=20d=C3=A9p=20de=20l'acheteu?= =?UTF-8?q?r)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/figures.py b/src/figures.py index f06ae37..008c1cf 100644 --- a/src/figures.py +++ b/src/figures.py @@ -11,20 +11,20 @@ from dash import dash_table, dcc, html from src.utils import data_schema, df, format_number -def get_map_count_marches(): - lf = df.lazy() - lf = lf.with_columns( - pl.col("lieuExecution_code").str.head(2).str.zfill(2).alias("Département") - ) - lf = ( - lf.select(["uid", "Département"]) +def get_map_count_marches(dff: pl.DataFrame) -> go.Figure: + lff: pl.LazyFrame = dff.lazy() + lff = lff.rename({"acheteur_departement_code": "Département"}) + + lff = ( + lff.select(["uid", "Département"]) .drop_nulls() - .unique(subset="uid") + .group_by("uid") + .agg(pl.col("Département").first()) .group_by("Département") .len("uid") ) # Suppression des infos pour les DOM/TOM pour l'instant - lf = lf.remove(pl.col("Département").is_in(["97", "98"])) + lff = lff.filter(~pl.col("Département").str.head(2).is_in(["97", "98"])) with open("./data/departements-1000m.geojson") as f: departements = json.load(f) @@ -33,7 +33,7 @@ def get_map_count_marches(): for f in departements["features"]: f["id"] = f["properties"]["code"] - df_map = lf.collect(engine="streaming") + df_map = lff.collect(engine="streaming") fig = px.choropleth( df_map, From 4771d14744b137ceecbb5c7e3f88ecb23ab8f59c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 14 Mar 2026 00:00:15 +0100 Subject: [PATCH 13/52] =?UTF-8?q?Utilise=20map=5Fcount=5Fmarches=20si=20tr?= =?UTF-8?q?op=20de=20march=C3=A9s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/figures.py b/src/figures.py index 008c1cf..c95be13 100644 --- a/src/figures.py +++ b/src/figures.py @@ -426,10 +426,18 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: Génère les cartes géographiques pour la métropole et les DOM-TOM. """ - # Seulement si les données ne sont pas trop importantes - - if dff.height > 10000: - return [] + # Si les données sont trop importantes on utilise une carte chloropleth + if dff.height > 5000: + return [ + dbc.Col( + dcc.Graph( + figure=get_map_count_marches(dff), config={"displayModeBar": False} + ), + width=12, + md=12, + className="mb-4", + ) + ] # Liste des codes départements Outre-Mer dom_codes = ["971", "972", "973", "974", "976"] From f0f9d8cb3df543f4f582a910595398c86a25ce74 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 14 Mar 2026 00:15:53 +0100 Subject: [PATCH 14/52] dashboard_acheteur_departement_code est une liste --- src/pages/statistiques.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 1876dfa..0237abb 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -146,10 +146,7 @@ def udpate_dashboard_cards( pl.col("acheteur_categorie").is_in(dashboard_acheteur_categorie) ) - if ( - dashboard_acheteur_departement_code - and len(dashboard_acheteur_departement_code) >= 2 - ): + if dashboard_acheteur_departement_code: lff = lff.filter( pl.col("acheteur_departement_code").is_in( dashboard_acheteur_departement_code From 302d253e6e9211aa5d5c3e8716a0f51d21ce2150 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sat, 14 Mar 2026 00:28:02 +0100 Subject: [PATCH 15/52] =?UTF-8?q?Utilisation=20de=20cluster=20de=20marqueu?= =?UTF-8?q?rs=20gr=C3=A2ce=20=C3=A0=20dash-leaflet=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 +- src/assets/dash_clientside.js | 23 ++++++ src/figures.py | 145 +++++++++++++++++++++------------- src/pages/statistiques.py | 4 +- 4 files changed, 116 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cdca66b..2befbcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,9 @@ dependencies = [ "plotly[express]", "httpx", "pandas", # utilisé pour la création de certains graphiques - "unidecode" + "unidecode", + "dash-leaflet", + "dash-extensions" ] [project.optional-dependencies] diff --git a/src/assets/dash_clientside.js b/src/assets/dash_clientside.js index efef901..babba1c 100644 --- a/src/assets/dash_clientside.js +++ b/src/assets/dash_clientside.js @@ -1,4 +1,27 @@ window.dash_clientside = Object.assign({}, window.dash_clientside, { + leaflet: { + pointToLayer: function (feature, latlng, context) { + return L.circleMarker(latlng, { + radius: 5, + fillColor: feature.properties.marker_color, + color: "white", + weight: 1, + opacity: 1, + fillOpacity: 0.8, + }).bindTooltip(feature.properties.tooltip); + }, + clusterToLayer: function (feature, latlng, index, context) { + const count = feature.properties.point_count; + const size = count < 100 ? 30 : count < 1000 ? 40 : 50; + const color = "#333"; // Default cluster color + const icon = L.divIcon({ + html: `
${count}
`, + className: "marker-cluster", + iconSize: L.point(size, size), + }); + return L.marker(latlng, { icon: icon }); + }, + }, clientside: { clean_filters: function (trigger) { if (!trigger) { diff --git a/src/figures.py b/src/figures.py index c95be13..15319b8 100644 --- a/src/figures.py +++ b/src/figures.py @@ -3,10 +3,13 @@ from typing import Literal from urllib.error import HTTPError, URLError import dash_bootstrap_components as dbc +import dash_leaflet as dl +import dash_leaflet.express as dlx import plotly.express as px import plotly.graph_objects as go import polars as pl from dash import dash_table, dcc, html +from dash_extensions.javascript import Namespace from src.utils import data_schema, df, format_number @@ -452,8 +455,8 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: # Ajout des DOM s'ils ont des données for code in dom_codes: - dom_data = dff.filter(pl.col("acheteur_departement_code") == code) - if dom_data.height > 0: + dff_dom = dff.filter(pl.col("acheteur_departement_code") == code) + if dff_dom.height > 0: name = f"Département {code}" if code == "971": name = "Guadeloupe" @@ -465,76 +468,106 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: name = "La Réunion" elif code == "976": name = "Mayotte" - regions[name] = dom_data + regions[name] = dff_dom + + # Region centers for dash-leaflet + region_centers = { + "Métropole": ([46.6, 2.2], 6), + "Guadeloupe": ([16.23, -61.55], 9), + "Martinique": ([14.64, -61.02], 10), + "Guyane": ([3.93, -53.12], 7), + "La Réunion": ([-21.11, 55.53], 10), + "Mayotte": ([-12.82, 45.16], 11), + } + + # JavaScript functions for styling + ns = Namespace("dash_clientside", "leaflet") + point_to_layer = ns("pointToLayer") + cluster_to_layer = ns("clusterToLayer") cols = [] for name, region_df in regions.items(): - fig = go.Figure() + # Prepare data for GeoJSON + marker_dicts = [] # Trace Acheteurs - mask_acheteur = region_df.filter( - pl.col("acheteur_latitude").is_not_null() - & pl.col("acheteur_longitude").is_not_null() - ) - if mask_acheteur.height > 0: - fig.add_trace( - go.Scattergeo( - lat=mask_acheteur["acheteur_latitude"], - lon=mask_acheteur["acheteur_longitude"], - mode="markers", - marker=dict(size=6, color=color_acheteur, opacity=0.5), - name="Acheteurs", - text=mask_acheteur["acheteur_nom"], - ) + mask_acheteur = ( + region_df.select( + "uid", "acheteur_longitude", "acheteur_latitude", "acheteur_nom" ) + .group_by("acheteur_longitude", "acheteur_latitude", "acheteur_nom") + .len("nb_marches") + .filter( + pl.col("acheteur_latitude").is_not_null() + & pl.col("acheteur_longitude").is_not_null() + ) + ) + + if mask_acheteur.height > 0: + for row in mask_acheteur.to_dicts(): + marker_dicts.append( + { + "lat": row["acheteur_latitude"], + "lon": row["acheteur_longitude"], + "tooltip": f"{row['acheteur_nom']} ({row['nb_marches']} marchés)", + "marker_color": color_acheteur, + } + ) # Trace Titulaires - mask_titulaire = region_df.filter( - pl.col("titulaire_latitude").is_not_null() - & pl.col("titulaire_longitude").is_not_null() - ) - if mask_titulaire.height > 0: - fig.add_trace( - go.Scattergeo( - lat=mask_titulaire["titulaire_latitude"], - lon=mask_titulaire["titulaire_longitude"], - mode="markers", - marker=dict(size=6, color=color_titulaire, opacity=0.5), - name="Titulaires", - text=mask_titulaire["titulaire_nom"], - ) + mask_titulaire = ( + region_df.select( + "uid", "titulaire_longitude", "titulaire_latitude", "titulaire_nom" + ) + .group_by("titulaire_longitude", "titulaire_latitude", "titulaire_nom") + .len("nb_marches") + .filter( + pl.col("titulaire_latitude").is_not_null() + & pl.col("titulaire_longitude").is_not_null() ) - - # Configuration spécifique de la vue - geo_config = dict( - projection_type="mercator", - showland=True, - landcolor="lightgray", - showcountries=True, - countrycolor="white", - fitbounds="locations" if name != "Métropole" else False, - resolution=50, ) - if name == "Métropole": - geo_config["lataxis_range"] = [41, 52] - geo_config["lonaxis_range"] = [-5, 10] + if mask_titulaire.height > 0: + for row in mask_titulaire.to_dicts(): + marker_dicts.append( + { + "lat": row["titulaire_latitude"], + "lon": row["titulaire_longitude"], + "tooltip": f"{row['titulaire_nom']} ({row['nb_marches']} marchés)", + "marker_color": color_titulaire, + } + ) - fig.update_layout( - title=name, - geo=geo_config, - margin=dict(l=0, r=0, t=30, b=0), - height=400 if name == "Métropole" else 300, - showlegend=(name == "Métropole"), - legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01), - ) + geojson_data = dlx.dicts_to_geojson(marker_dicts) - # Taille de la colonne : 4 slots (md=12) pour Métropole, 1 slot (md=3) pour DOM - # On suppose une grille de 12 colonnes où 1 card = md=3 + center, zoom = region_centers.get(name, ([46.6, 2.2], 6)) col_width = 12 if name == "Métropole" else 3 + region_id = name.lower().replace(" ", "-") cols.append( dbc.Col( - dcc.Graph(figure=fig, config={"displayModeBar": False}), + [ + html.H5(name), + dl.Map( + [ + dl.TileLayer(), + dl.GeoJSON( + data=geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}", + ), + ], + center=center, + zoom=zoom, + style={ + "width": "100%", + "height": "400px" if name == "Métropole" else "300px", + }, + id=f"map-{region_id}", + ), + ], width=12, md=col_width, className="mb-4", diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 0237abb..1a74231 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -142,9 +142,7 @@ def udpate_dashboard_cards( ) if dashboard_acheteur_categorie: - lff = lff.filter( - pl.col("acheteur_categorie").is_in(dashboard_acheteur_categorie) - ) + lff = lff.filter(pl.col("acheteur_categorie") == dashboard_acheteur_categorie) if dashboard_acheteur_departement_code: lff = lff.filter( From adc8457abccfbe50f7cbea9873f282efda678962 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Sun, 15 Mar 2026 16:05:33 +0100 Subject: [PATCH 16/52] Cartes avec cluster de points si > lignes #65 --- src/assets/dash_clientside.js | 8 +++- src/figures.py | 70 ++++++++++++++++++++++++----------- src/pages/statistiques.py | 1 + 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/assets/dash_clientside.js b/src/assets/dash_clientside.js index babba1c..2c41e6a 100644 --- a/src/assets/dash_clientside.js +++ b/src/assets/dash_clientside.js @@ -11,11 +11,15 @@ window.dash_clientside = Object.assign({}, window.dash_clientside, { }).bindTooltip(feature.properties.tooltip); }, clusterToLayer: function (feature, latlng, index, context) { + console.log(feature); + console.log(index); + console.log(context); + const count = feature.properties.point_count; const size = count < 100 ? 30 : count < 1000 ? 40 : 50; - const color = "#333"; // Default cluster color + const color = "#555"; // Default cluster color const icon = L.divIcon({ - html: `
${count}
`, + html: `
${count}
`, className: "marker-cluster", iconSize: L.point(size, size), }); diff --git a/src/figures.py b/src/figures.py index 15319b8..a63cab8 100644 --- a/src/figures.py +++ b/src/figures.py @@ -430,7 +430,7 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: """ # Si les données sont trop importantes on utilise une carte chloropleth - if dff.height > 5000: + if dff.height > 50000: return [ dbc.Col( dcc.Graph( @@ -443,21 +443,30 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: ] # Liste des codes départements Outre-Mer - dom_codes = ["971", "972", "973", "974", "976"] + region_codes: list = ["Métropole", "971", "972", "973", "974", "976"] + dom_codes = region_codes[1:] # Couleurs accessibles (Okabe-Ito) color_acheteur = "#E69F00" # Orange color_titulaire = "#56B4E9" # Bleu ciel - regions = { - "Métropole": dff.filter(~pl.col("acheteur_departement_code").is_in(dom_codes)) - } + regions = {} # Ajout des DOM s'ils ont des données - for code in dom_codes: - dff_dom = dff.filter(pl.col("acheteur_departement_code") == code) - if dff_dom.height > 0: - name = f"Département {code}" + for code in region_codes: + if code == "Métropole": + dff_region = dff.filter( + ( + ~pl.col("acheteur_departement_code").is_in(dom_codes) + | (~pl.col("titulaire_departement_code").is_in(dom_codes)) + ) + ) + else: + dff_region = dff.filter( + (pl.col("acheteur_departement_code") == code) + | (pl.col("titulaire_departement_code") == code) + ) + if dff_region.height > 0: if code == "971": name = "Guadeloupe" elif code == "972": @@ -468,16 +477,21 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: name = "La Réunion" elif code == "976": name = "Mayotte" - regions[name] = dff_dom + elif code == "Métropole": + name = "Métropole" + else: + name = f"Département {code}" + + regions[name] = dff_region # Region centers for dash-leaflet region_centers = { - "Métropole": ([46.6, 2.2], 6), + "Métropole": ([46.6, 2.2], 5), "Guadeloupe": ([16.23, -61.55], 9), "Martinique": ([14.64, -61.02], 10), "Guyane": ([3.93, -53.12], 7), - "La Réunion": ([-21.11, 55.53], 10), - "Mayotte": ([-12.82, 45.16], 11), + "La Réunion": ([-21.11, 55.53], 9), + "Mayotte": ([-12.82, 45.16], 10), } # JavaScript functions for styling @@ -487,9 +501,6 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: cols = [] for name, region_df in regions.items(): - # Prepare data for GeoJSON - marker_dicts = [] - # Trace Acheteurs mask_acheteur = ( region_df.select( @@ -503,9 +514,11 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: ) ) + acheteurs_marker_dicts = [] + if mask_acheteur.height > 0: for row in mask_acheteur.to_dicts(): - marker_dicts.append( + acheteurs_marker_dicts.append( { "lat": row["acheteur_latitude"], "lon": row["acheteur_longitude"], @@ -527,9 +540,11 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: ) ) + titulaires_marker_dicts = [] + if mask_titulaire.height > 0: for row in mask_titulaire.to_dicts(): - marker_dicts.append( + titulaires_marker_dicts.append( { "lat": row["titulaire_latitude"], "lon": row["titulaire_longitude"], @@ -538,10 +553,11 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: } ) - geojson_data = dlx.dicts_to_geojson(marker_dicts) + acheteurs_geojson_data = dlx.dicts_to_geojson(acheteurs_marker_dicts) + titulaires_geojson_data = dlx.dicts_to_geojson(titulaires_marker_dicts) center, zoom = region_centers.get(name, ([46.6, 2.2], 6)) - col_width = 12 if name == "Métropole" else 3 + col_width = 6 if name == "Métropole" else 3 region_id = name.lower().replace(" ", "-") cols.append( dbc.Col( @@ -551,12 +567,22 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: [ dl.TileLayer(), dl.GeoJSON( - data=geojson_data, + data=titulaires_geojson_data, cluster=True, zoomToBoundsOnClick=True, pointToLayer=point_to_layer, clusterToLayer=cluster_to_layer, - id=f"geojson-{region_id}", + id=f"geojson-{region_id}-titulaires", + options={"fillColor": color_titulaire}, + ), + dl.GeoJSON( + data=acheteurs_geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}-acheteurs", + options={"fillColor": color_acheteur}, ), ], center=center, diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 1a74231..99fcbc7 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -163,6 +163,7 @@ def udpate_dashboard_cards( total_montant = df_per_uid.select(pl.col("montant").sum()).item() nb_marches = df_per_uid.height + # À transformer en fonction card_basic_counts = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), html.P( From f531ce70918fa543b38750e4f0c9f61f2e14360a Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Mon, 16 Mar 2026 18:06:32 +0100 Subject: [PATCH 17/52] =?UTF-8?q?Choix=20de=20carte=20dynamique,=20m=C3=AA?= =?UTF-8?q?me=20pour=20les=20TOM=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 416 +++++++++++++++++++++++++------------------------ src/utils.py | 12 ++ 2 files changed, 226 insertions(+), 202 deletions(-) diff --git a/src/figures.py b/src/figures.py index a63cab8..1d3a132 100644 --- a/src/figures.py +++ b/src/figures.py @@ -1,4 +1,3 @@ -import json from typing import Literal from urllib.error import HTTPError, URLError @@ -11,57 +10,7 @@ import polars as pl from dash import dash_table, dcc, html from dash_extensions.javascript import Namespace -from src.utils import data_schema, df, format_number - - -def get_map_count_marches(dff: pl.DataFrame) -> go.Figure: - lff: pl.LazyFrame = dff.lazy() - lff = lff.rename({"acheteur_departement_code": "Département"}) - - lff = ( - lff.select(["uid", "Département"]) - .drop_nulls() - .group_by("uid") - .agg(pl.col("Département").first()) - .group_by("Département") - .len("uid") - ) - # Suppression des infos pour les DOM/TOM pour l'instant - lff = lff.filter(~pl.col("Département").str.head(2).is_in(["97", "98"])) - - with open("./data/departements-1000m.geojson") as f: - departements = json.load(f) - - # Ajout de feature.id - for f in departements["features"]: - f["id"] = f["properties"]["code"] - - df_map = lff.collect(engine="streaming") - - fig = px.choropleth( - df_map, - geojson=departements, - locations="Département", - color="uid", - color_continuous_scale="Reds", - title="Nombres de marchés attribués par département (lieu d'exécution)", - range_color=(df_map["uid"].min(), df_map["uid"].max()), - labels={"uid": "Marchés attribués"}, - scope="europe", - width=900, - height=700, - ) - - fig.update_geos(fitbounds="locations", visible=False) - fig.update_layout( - mapbox={ - "style": "carto-positron", - "center": {"lon": 10, "lat": 10}, - "zoom": 1, - "domain": {"x": [0, 1], "y": [0, 1]}, - } - ) - return fig +from src.utils import data_schema, departements_geojson, df, format_number def get_yearly_statistics(statistics, today_str) -> html.Div: @@ -82,11 +31,11 @@ def get_yearly_statistics(statistics, today_str) -> html.Div: } ) - df = pl.DataFrame(data) + dff = pl.DataFrame(data) # Create Dash DataTable table = dash_table.DataTable( - data=df.to_dicts(), + data=dff.to_dicts(), columns=[ {"name": "Année", "id": "Année"}, {"name": "Marchés et accord-cadres", "id": "Marchés et accord-cadres"}, @@ -429,178 +378,241 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: Génère les cartes géographiques pour la métropole et les DOM-TOM. """ - # Si les données sont trop importantes on utilise une carte chloropleth - if dff.height > 50000: - return [ - dbc.Col( - dcc.Graph( - figure=get_map_count_marches(dff), config={"displayModeBar": False} - ), - width=12, - md=12, - className="mb-4", - ) - ] + regions: dict = { + "Métropole": { + "coordinates": [46.6, 2.2], + "zoom_leaflet": 5, + "zoom_chloropleth": 1, + "name": "Métropole", + }, + "971": { + "coordinates": [16.23, -61.55], + "zoom_leaflet": 9, + "zoom_chloropleth": 1, + "name": "Guadeloupe", + }, + "972": { + "coordinates": [14.64, -61.02], + "zoom_leaflet": 10, + "zoom_chloropleth": 1, + "name": "Martinique", + }, + "973": { + "coordinates": [3.93, -53.12], + "zoom_leaflet": 7, + "zoom_chloropleth": 1, + "name": "Guyane", + }, + "974": { + "coordinates": [-21.11, 55.53], + "zoom_leaflet": 9, + "zoom_chloropleth": 1, + "name": "La Réunion", + }, + "976": { + "coordinates": [-12.82, 45.16], + "zoom_leaflet": 10, + "zoom_chloropleth": 1, + "name": "Mayotte", + }, + } # Liste des codes départements Outre-Mer - region_codes: list = ["Métropole", "971", "972", "973", "974", "976"] - dom_codes = region_codes[1:] + dom_codes = [code for code in regions.keys() if code != "Métropole"] + print("dom codes", dom_codes) - # Couleurs accessibles (Okabe-Ito) - color_acheteur = "#E69F00" # Orange - color_titulaire = "#56B4E9" # Bleu ciel - - regions = {} - - # Ajout des DOM s'ils ont des données - for code in region_codes: - if code == "Métropole": - dff_region = dff.filter( + def make_map_data(region_code: str) -> tuple[list, str or None]: + lff: pl.LazyFrame = dff.lazy() + if region_code == "Métropole": + lff = lff.filter( ( ~pl.col("acheteur_departement_code").is_in(dom_codes) | (~pl.col("titulaire_departement_code").is_in(dom_codes)) ) ) else: - dff_region = dff.filter( + lff = lff.filter( (pl.col("acheteur_departement_code") == code) | (pl.col("titulaire_departement_code") == code) ) - if dff_region.height > 0: - if code == "971": - name = "Guadeloupe" - elif code == "972": - name = "Martinique" - elif code == "973": - name = "Guyane" - elif code == "974": - name = "La Réunion" - elif code == "976": - name = "Mayotte" - elif code == "Métropole": - name = "Métropole" - else: - name = f"Département {code}" - regions[name] = dff_region + nb_marches = lff.select("uid").group_by("uid").first().collect().height - # Region centers for dash-leaflet - region_centers = { - "Métropole": ([46.6, 2.2], 5), - "Guadeloupe": ([16.23, -61.55], 9), - "Martinique": ([14.64, -61.02], 10), - "Guyane": ([3.93, -53.12], 7), - "La Réunion": ([-21.11, 55.53], 9), - "Mayotte": ([-12.82, 45.16], 10), - } + if nb_marches == 0: + return [], None + dfs = [] + + if (code == "Métropole" and nb_marches > 20000) or ( + code != "Métropole" and nb_marches > 10000 + ): + _map_type: str = "chloropleth" + + lff = lff.rename({"acheteur_departement_code": "Département"}) + lff = ( + lff.select(["uid", "Département"]) + .drop_nulls() + .group_by("uid") + .agg(pl.col("Département").first()) + .group_by("Département") + .len("uid") + ) + dfs.append(lff.collect()) + else: + _map_type: str = "clusters" + for org_type in ["acheteur", "titulaire"]: + lff_org = ( + lff.select( + "uid", + f"{org_type}_longitude", + f"{org_type}_latitude", + f"{org_type}_nom", + ) + .group_by( + f"{org_type}_longitude", + f"{org_type}_latitude", + f"{org_type}_nom", + ) + .len("nb_marches") + .filter( + pl.col(f"{org_type}_latitude").is_not_null() + & pl.col(f"{org_type}_longitude").is_not_null() + ) + ) + + markers = [] + + # Couleurs accessibles (Okabe-Ito) + colors = { + "acheteur": "#E69F00", # orange + "titulaire": "#56B4E9", # bleu ciel + } + + for row in lff_org.collect().to_dicts(): + markers.append( + { + "lat": row[f"{org_type}_latitude"], + "lon": row[f"{org_type}_longitude"], + "tooltip": f"{row[f'{org_type}_nom']} ({row['nb_marches']} marchés)", + "marker_color": colors[org_type], + } + ) + dfs.append(markers) + + return dfs, _map_type + + cols = [] + + for code in regions.keys(): + regions[code]["data"], map_type = make_map_data(code) + print("region", regions[code]["name"], len(regions[code]["data"][0]), map_type) + + if map_type == "chloropleth": + map_graph = make_chloropleth_map(regions[code]) # call chloropleth function + elif map_type == "clusters": + map_graph = make_clusters_map(regions[code]) + elif map_type is None: + continue + else: + raise ValueError(f"Map type '{map_type}' not recognised") + + col = dbc.Col( + children=[ + html.H5(regions[code]["name"]), + map_graph, + ], + md=6 if code == "Métropole" else 3, + width=12, + className="mb-4", + ) + cols.append(col) + + return cols + + +def make_chloropleth_map(region: dict) -> dcc.Graph: + df_map = region["data"][0] + + fig = px.choropleth( + df_map, + geojson=departements_geojson, + locations="Département", + color="uid", + color_continuous_scale="Reds", + title="Nombres de marchés attribués", + range_color=(df_map["uid"].min(), df_map["uid"].max()), + labels={"uid": "Marchés attribués"}, + scope="europe", + width=400, + height=400, + ) + + fig.update_geos(fitbounds="locations", visible=False) + fig.update_layout( + mapbox={ + "style": "carto-positron", + "center": {"lon": 10, "lat": 10}, + "zoom": 8, + "domain": {"x": [0, 1], "y": [0, 1]}, + } + ) + + graph = dcc.Graph(figure=fig, config={"displayModeBar": False}) + return graph + + +def make_clusters_map(region: dict): # JavaScript functions for styling ns = Namespace("dash_clientside", "leaflet") point_to_layer = ns("pointToLayer") cluster_to_layer = ns("clusterToLayer") - cols = [] - for name, region_df in regions.items(): - # Trace Acheteurs - mask_acheteur = ( - region_df.select( - "uid", "acheteur_longitude", "acheteur_latitude", "acheteur_nom" - ) - .group_by("acheteur_longitude", "acheteur_latitude", "acheteur_nom") - .len("nb_marches") - .filter( - pl.col("acheteur_latitude").is_not_null() - & pl.col("acheteur_longitude").is_not_null() - ) - ) + name = region["name"] - acheteurs_marker_dicts = [] + # Données de la région + region_acheteurs = region["data"][0] + region_titulaires = region["data"][1] - if mask_acheteur.height > 0: - for row in mask_acheteur.to_dicts(): - acheteurs_marker_dicts.append( - { - "lat": row["acheteur_latitude"], - "lon": row["acheteur_longitude"], - "tooltip": f"{row['acheteur_nom']} ({row['nb_marches']} marchés)", - "marker_color": color_acheteur, - } - ) + # Couleurs + color_acheteur = region_acheteurs[0]["marker_color"] + color_titulaire = region_titulaires[0]["marker_color"] - # Trace Titulaires - mask_titulaire = ( - region_df.select( - "uid", "titulaire_longitude", "titulaire_latitude", "titulaire_nom" - ) - .group_by("titulaire_longitude", "titulaire_latitude", "titulaire_nom") - .len("nb_marches") - .filter( - pl.col("titulaire_latitude").is_not_null() - & pl.col("titulaire_longitude").is_not_null() - ) - ) + acheteurs_geojson_data = dlx.dicts_to_geojson(region_acheteurs) + titulaires_geojson_data = dlx.dicts_to_geojson(region_titulaires) - titulaires_marker_dicts = [] - - if mask_titulaire.height > 0: - for row in mask_titulaire.to_dicts(): - titulaires_marker_dicts.append( - { - "lat": row["titulaire_latitude"], - "lon": row["titulaire_longitude"], - "tooltip": f"{row['titulaire_nom']} ({row['nb_marches']} marchés)", - "marker_color": color_titulaire, - } - ) - - acheteurs_geojson_data = dlx.dicts_to_geojson(acheteurs_marker_dicts) - titulaires_geojson_data = dlx.dicts_to_geojson(titulaires_marker_dicts) - - center, zoom = region_centers.get(name, ([46.6, 2.2], 6)) - col_width = 6 if name == "Métropole" else 3 - region_id = name.lower().replace(" ", "-") - cols.append( - dbc.Col( - [ - html.H5(name), - dl.Map( - [ - dl.TileLayer(), - dl.GeoJSON( - data=titulaires_geojson_data, - cluster=True, - zoomToBoundsOnClick=True, - pointToLayer=point_to_layer, - clusterToLayer=cluster_to_layer, - id=f"geojson-{region_id}-titulaires", - options={"fillColor": color_titulaire}, - ), - dl.GeoJSON( - data=acheteurs_geojson_data, - cluster=True, - zoomToBoundsOnClick=True, - pointToLayer=point_to_layer, - clusterToLayer=cluster_to_layer, - id=f"geojson-{region_id}-acheteurs", - options={"fillColor": color_acheteur}, - ), - ], - center=center, - zoom=zoom, - style={ - "width": "100%", - "height": "400px" if name == "Métropole" else "300px", - }, - id=f"map-{region_id}", - ), - ], - width=12, - md=col_width, - className="mb-4", - ) - ) - - return cols + center, zoom = region["coordinates"], region["zoom_leaflet"] + region_id = name.lower().replace(" ", "-") + leaflet_map = dl.Map( + [ + dl.TileLayer(), + dl.GeoJSON( + data=titulaires_geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}-titulaires", + options={"fillColor": color_titulaire}, + ), + dl.GeoJSON( + data=acheteurs_geojson_data, + cluster=True, + zoomToBoundsOnClick=True, + pointToLayer=point_to_layer, + clusterToLayer=cluster_to_layer, + id=f"geojson-{region_id}-acheteurs", + options={"fillColor": color_acheteur}, + ), + ], + center=center, + zoom=zoom, + style={ + "width": "100%", + "height": "400px" if name == "Métropole" else "300px", + }, + id=f"map-{region_id}", + ) + return leaflet_map def make_column_picker(page: str): diff --git a/src/utils.py b/src/utils.py index 992a99b..b334d01 100644 --- a/src/utils.py +++ b/src/utils.py @@ -286,6 +286,17 @@ def get_departements() -> dict: return data +def get_departements_geojson() -> dict: + with open("./data/departements-1000m.geojson") as f: + geojson = json.load(f) + + # Ajout de feature.id + for f in geojson["features"]: + f["id"] = f["properties"]["code"] + + return geojson + + def get_departement_region(code_postal): if code_postal > "97000": code_departement = code_postal[:3] @@ -774,6 +785,7 @@ df_titulaires_marches: pl.DataFrame = ( ) departements = get_departements() +departements_geojson = get_departements_geojson() domain_name = ( "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" ) From ab3377ef60ef39804188a4ff56b96c86d2a4ba42 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 17:16:24 +0100 Subject: [PATCH 18/52] Grid de cards avec points de rupture #65 --- data/departements.json | 8 ++++++ src/figures.py | 57 +++++++++++++++++++++++---------------- src/pages/statistiques.py | 37 +++++++++---------------- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/data/departements.json b/data/departements.json index 4115895..1f346d0 100644 --- a/data/departements.json +++ b/data/departements.json @@ -391,8 +391,16 @@ "departement": "La Réunion", "region": "La Réunion" }, + "975": { + "departement": "Saint-Pierre-et-Miquelon", + "region": "Saint-Pierre-et-Miquelon" + }, "976": { "departement": "Mayotte", "region": "Mayotte" + }, + "977": { + "departement": "Saint-Barthelemy", + "region": "Saint-Barthelemy" } } diff --git a/src/figures.py b/src/figures.py index 1d3a132..556e802 100644 --- a/src/figures.py +++ b/src/figures.py @@ -417,18 +417,12 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: }, } - # Liste des codes départements Outre-Mer - dom_codes = [code for code in regions.keys() if code != "Métropole"] - print("dom codes", dom_codes) - def make_map_data(region_code: str) -> tuple[list, str or None]: lff: pl.LazyFrame = dff.lazy() if region_code == "Métropole": lff = lff.filter( - ( - ~pl.col("acheteur_departement_code").is_in(dom_codes) - | (~pl.col("titulaire_departement_code").is_in(dom_codes)) - ) + (pl.col("acheteur_departement_code").str.len_chars() == 2) + & (pl.col("titulaire_departement_code").str.len_chars() == 2) ) else: lff = lff.filter( @@ -436,14 +430,14 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: | (pl.col("titulaire_departement_code") == code) ) - nb_marches = lff.select("uid").group_by("uid").first().collect().height + nb_marches = lff.select("uid").collect()["uid"].n_unique() if nb_marches == 0: return [], None dfs = [] - if (code == "Métropole" and nb_marches > 20000) or ( + if (code == "Métropole" and nb_marches > 30000) or ( code != "Métropole" and nb_marches > 10000 ): _map_type: str = "chloropleth" @@ -505,10 +499,9 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: for code in regions.keys(): regions[code]["data"], map_type = make_map_data(code) - print("region", regions[code]["name"], len(regions[code]["data"][0]), map_type) if map_type == "chloropleth": - map_graph = make_chloropleth_map(regions[code]) # call chloropleth function + map_graph = make_chloropleth_map(regions[code]) elif map_type == "clusters": map_graph = make_clusters_map(regions[code]) elif map_type is None: @@ -516,15 +509,9 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: else: raise ValueError(f"Map type '{map_type}' not recognised") - col = dbc.Col( - children=[ - html.H5(regions[code]["name"]), - map_graph, - ], - md=6 if code == "Métropole" else 3, - width=12, - className="mb-4", - ) + md = 6 if code == "Métropole" else 3 + + col = make_card(regions[code]["name"], md=md, fig=map_graph) cols.append(col) return cols @@ -543,7 +530,6 @@ def make_chloropleth_map(region: dict) -> dcc.Graph: range_color=(df_map["uid"].min(), df_map["uid"].max()), labels={"uid": "Marchés attribués"}, scope="europe", - width=400, height=400, ) @@ -561,7 +547,7 @@ def make_chloropleth_map(region: dict) -> dcc.Graph: return graph -def make_clusters_map(region: dict): +def make_clusters_map(region: dict) -> dl.Map: # JavaScript functions for styling ns = Namespace("dash_clientside", "leaflet") point_to_layer = ns("pointToLayer") @@ -615,6 +601,31 @@ def make_clusters_map(region: dict): return leaflet_map +def make_card( + title: str, subtitle=None, fig=None, paragraphs=None, md=3, width=12 +) -> dbc.Col: + children = [] + if title: + children.append(html.H5(title, className="card-title")) + if subtitle: + children.append(html.H6(subtitle, className="card-subtitle mb-2 text-muted")) + if fig: + children.append(fig) + if paragraphs: + for p in paragraphs: + p.className = "card-text" + children.append(p) + + card = dbc.Col( + html.Div(html.Div(className="card-body", children=children), className="card"), + lg=6, + xl=4, + # width=width, + # className="mb-4", + ) + return card + + def make_column_picker(page: str): table_data = [] table_columns = [ diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 99fcbc7..df71edd 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -5,9 +5,7 @@ import polars as pl import polars.selectors as cs from dash import Input, Output, callback, dcc, html, register_page -from src.figures import ( - get_geographic_maps, -) +from src.figures import get_geographic_maps, make_card from src.utils import ( departements, df, @@ -52,8 +50,8 @@ layout = [ dbc.Row( [ dbc.Col( - width=12, - md=3, + xl=3, + lg=4, id="filters", children=[ html.H5("Période d'attribution"), @@ -87,22 +85,10 @@ layout = [ ), dbc.Col( width=12, - md=9, + lg=8, + xl=9, id="cards", - children=[ - dbc.Row( - ( - dbc.Col( - width=6, - md=4, - className="card", - id="card_basic_counts", - ) - ), - className="mb-4", - ), - dbc.Row(id="maps_row"), - ], + children=[], ), ] ) @@ -114,8 +100,7 @@ layout = [ @callback( - Output("card_basic_counts", "children"), - Output("maps_row", "children"), + Output("cards", "children"), Input("dashboard_year", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), @@ -163,6 +148,8 @@ def udpate_dashboard_cards( total_montant = df_per_uid.select(pl.col("montant").sum()).item() nb_marches = df_per_uid.height + cards = [] + # À transformer en fonction card_basic_counts = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), @@ -175,6 +162,8 @@ def udpate_dashboard_cards( html.P(["Montant total : ", html.Strong(format_number(total_montant) + " €")]), ] - geographic_maps = get_geographic_maps(dff) + cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) - return card_basic_counts, geographic_maps + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) + + return dbc.Row(children=cards + geographic_maps) From 9629231e299a4876370080450b0685d917c9f0d4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 17:30:41 +0100 Subject: [PATCH 19/52] Fixed masquage silencieux de la chloropleth #65 --- src/figures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/figures.py b/src/figures.py index 556e802..0f7f6b0 100644 --- a/src/figures.py +++ b/src/figures.py @@ -609,7 +609,7 @@ def make_card( children.append(html.H5(title, className="card-title")) if subtitle: children.append(html.H6(subtitle, className="card-subtitle mb-2 text-muted")) - if fig: + if fig is not None: children.append(fig) if paragraphs: for p in paragraphs: From e754a3a21784cf470b52c2e33fd9786dadbf4fdd Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 20:33:36 +0100 Subject: [PATCH 20/52] Configuration fine des tailles de cartes #65 --- src/assets/css/style.css | 5 +++++ src/figures.py | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index a183642..326523f 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -423,6 +423,11 @@ input[type="checkbox"] { } /* --- Organization Cards (Grid Items) --- */ + +#cards .card { + margin-bottom: 16px; +} + .org_title { grid-column: 1 / 3; grid-row: 1; diff --git a/src/figures.py b/src/figures.py index 0f7f6b0..215847f 100644 --- a/src/figures.py +++ b/src/figures.py @@ -509,9 +509,9 @@ def get_geographic_maps(dff: pl.DataFrame) -> list | None: else: raise ValueError(f"Map type '{map_type}' not recognised") - md = 6 if code == "Métropole" else 3 + lg, xl = (12, 8) if code == "Métropole" else (6, 4) - col = make_card(regions[code]["name"], md=md, fig=map_graph) + col = make_card(regions[code]["name"], fig=map_graph, lg=lg, xl=xl) cols.append(col) return cols @@ -526,11 +526,9 @@ def make_chloropleth_map(region: dict) -> dcc.Graph: locations="Département", color="uid", color_continuous_scale="Reds", - title="Nombres de marchés attribués", range_color=(df_map["uid"].min(), df_map["uid"].max()), labels={"uid": "Marchés attribués"}, scope="europe", - height=400, ) fig.update_geos(fitbounds="locations", visible=False) @@ -602,7 +600,7 @@ def make_clusters_map(region: dict) -> dl.Map: def make_card( - title: str, subtitle=None, fig=None, paragraphs=None, md=3, width=12 + title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4 ) -> dbc.Col: children = [] if title: @@ -618,8 +616,8 @@ def make_card( card = dbc.Col( html.Div(html.Div(className="card-body", children=children), className="card"), - lg=6, - xl=4, + lg=lg, + xl=xl, # width=width, # className="mb-4", ) From 6c778882f94774f5bcf558810f4a13eb30dd2e92 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 20:51:09 +0100 Subject: [PATCH 21/52] =?UTF-8?q?Filtre=20par=20type=20de=20march=C3=A9=20?= =?UTF-8?q?#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/statistiques.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index df71edd..4a1f970 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -81,6 +81,14 @@ layout = [ options=options_departements, ), ), + html.H5("Marché"), + dbc.Row( + dcc.Dropdown( + id="dashboard_marche_type", + placeholder="Type de marché", + options=get_enum_values_as_dict("type"), + ), + ), ], ), dbc.Col( @@ -104,9 +112,13 @@ layout = [ Input("dashboard_year", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_marche_type", "value"), ) def udpate_dashboard_cards( - dashboard_year, dashboard_acheteur_categorie, dashboard_acheteur_departement_code + dashboard_year, + dashboard_acheteur_categorie, + dashboard_acheteur_departement_code, + dashboard_marche_type, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -136,6 +148,9 @@ def udpate_dashboard_cards( ) ) + if dashboard_marche_type: + lff = lff.filter(pl.col("type") == dashboard_marche_type) + # Génération des métriques dff = lff.collect() From c02fb995c54eeb05e39d83abd1f070a719bc3eb4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 22:44:15 +0100 Subject: [PATCH 22/52] =?UTF-8?q?Ajout=20de=20filtres=20et=20des=20donuts?= =?UTF-8?q?=20de=20cat=C3=A9gorie=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 7 ++++ src/figures.py | 19 +++++++++ src/pages/statistiques.py | 87 ++++++++++++++++++++++++++++++++++++--- src/utils.py | 14 +++---- 4 files changed, 114 insertions(+), 13 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 326523f..9705af4 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -190,6 +190,13 @@ p.version > a { grid-row: 1; } +/* --- Dashboard inputs --- */ + +.Select--multi .Select-value { + color: var(--primary-color); + background-color: rgba(255, 240, 240, 0.4); +} + /* --- Tables (Dash & Custom) --- */ /* Table Menu (Exports etc) */ diff --git a/src/figures.py b/src/figures.py index 215847f..c32f865 100644 --- a/src/figures.py +++ b/src/figures.py @@ -624,6 +624,25 @@ def make_card( return card +def make_donut(lff: pl.LazyFrame, names_col): + title = data_schema[names_col]["title"] + lff = lff.rename({names_col: title}) + lff = lff.select("uid", title) + lff = lff.group_by(title).len("Nombre") + lff = lff.with_columns(pl.col(title).replace(None, pl.lit("?"))) + fig = px.pie( + lff.collect(engine="streaming"), + values="Nombre", + names=title, + hole=0.4, + color_discrete_sequence=px.colors.qualitative.Safe, + ) + fig = fig.update_traces(texttemplate="%{label}
%{percent}") + fig = fig.update_layout(showlegend=False, font=dict(size=14)) + graph = dcc.Graph(figure=fig) + return graph + + def make_column_picker(page: str): table_data = [] table_columns = [ diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index 4a1f970..9de6a29 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -5,7 +5,7 @@ import polars as pl import polars.selectors as cs from dash import Input, Output, callback, dcc, html, register_page -from src.figures import get_geographic_maps, make_card +from src.figures import get_geographic_maps, make_card, make_donut from src.utils import ( departements, df, @@ -69,7 +69,7 @@ layout = [ options=get_enum_values_as_dict( "acheteur_categorie" ), - placeholder="Catégorie d'acheteur", + placeholder="Catégorie", ), ), dbc.Row( @@ -81,14 +81,44 @@ layout = [ options=options_departements, ), ), + html.H5("Titulaire"), + dbc.Row( + dcc.Dropdown( + id="dashboard_titulaire_categorie", + placeholder="Catégorie", + options=get_enum_values_as_dict( + "titulaire_categorie" + ), + ), + ), html.H5("Marché"), dbc.Row( dcc.Dropdown( id="dashboard_marche_type", - placeholder="Type de marché", + placeholder="Type", options=get_enum_values_as_dict("type"), ), ), + dbc.Row( + dcc.Dropdown( + id="dashboard_marche_considerationsSociales", + placeholder="Considérations sociales", + options=get_enum_values_as_dict( + "considerationsSociales" + ), + multi=True, + ), + ), + dbc.Row( + dcc.Dropdown( + id="dashboard_marche_considerationsEnvironnementales", + placeholder="Considérations environnementales", + multi=True, + options=get_enum_values_as_dict( + "considerationsEnvironnementales" + ), + ), + ), ], ), dbc.Col( @@ -112,13 +142,19 @@ layout = [ Input("dashboard_year", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_titulaire_categorie", "value"), Input("dashboard_marche_type", "value"), + Input("dashboard_marche_considerationsSociales", "value"), + Input("dashboard_marche_considerationsEnvironnementales", "value"), ) def udpate_dashboard_cards( dashboard_year, dashboard_acheteur_categorie, dashboard_acheteur_departement_code, + dashboard_titulaire_categorie, dashboard_marche_type, + dashboard_marche_considerationsSociales, + dashboard_marche_considerationsEnvironnementales, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -127,10 +163,14 @@ def udpate_dashboard_cards( cs.starts_with("titulaire"), "dateNotification", "montant", + "considerationsSociales", + "considerationsEnvironnementales", ) # Application des filtres + ## Période + if dashboard_year: lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) else: @@ -138,6 +178,8 @@ def udpate_dashboard_cards( pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) ) + ## Acheteur + if dashboard_acheteur_categorie: lff = lff.filter(pl.col("acheteur_categorie") == dashboard_acheteur_categorie) @@ -148,12 +190,38 @@ def udpate_dashboard_cards( ) ) + ## Titulaire + + if dashboard_titulaire_categorie: + lff = lff.filter(pl.col("titulaire_categorie") == dashboard_titulaire_categorie) + + ## Marché + if dashboard_marche_type: lff = lff.filter(pl.col("type") == dashboard_marche_type) - # Génération des métriques - dff = lff.collect() + if dashboard_marche_considerationsSociales: + lff = lff.filter( + pl.col("considerationsSociales") + .str.split(", ") + .list.set_intersection(dashboard_marche_considerationsSociales) + .list.len() + > 0 + ) + if dashboard_marche_considerationsEnvironnementales: + lff = lff.filter( + pl.col("considerationsEnvironnementales") + .str.split(", ") + .list.set_intersection(dashboard_marche_considerationsEnvironnementales) + .list.len() + > 0 + ) + + # Génération des métriques + dff = lff.collect(engine="streaming") + + # À transformer en fonction nb_acheteurs = dff.select("acheteur_id").n_unique() nb_titulaires = dff.select("titulaire_id", "titulaire_typeIdentifiant").n_unique() @@ -165,7 +233,6 @@ def udpate_dashboard_cards( cards = [] - # À transformer en fonction card_basic_counts = [ html.P(["Nombre de marchés : ", html.Strong(str(format_number(nb_marches)))]), html.P( @@ -179,6 +246,14 @@ def udpate_dashboard_cards( cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) + donut_acheteur_categorie = make_donut(lff, "acheteur_categorie") + cards.append(make_card(title="Catégorie d'acheteur", fig=donut_acheteur_categorie)) + + donut_titulaire_categorie = make_donut(lff, "titulaire_categorie") + cards.append( + make_card(title="Catégorie d'entreprise", fig=donut_titulaire_categorie) + ) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) return dbc.Row(children=cards + geographic_maps) diff --git a/src/utils.py b/src/utils.py index b334d01..642bad4 100644 --- a/src/utils.py +++ b/src/utils.py @@ -697,13 +697,13 @@ def get_button_properties(height): def get_enum_values_as_dict(column_name): - for column in data_schema: - if column == column_name: - options = {} - for value in data_schema[column]["enum"]: - options[value] = value - return options - return {"not_found": "not found"} + try: + options = {} + for value in data_schema[column_name]["enum"]: + options[value] = value + return options + except KeyError: + return {"not_found": "not found"} def invert_columns(columns): From d50ec5b01e274e80d7f2219515175515dfc920b7 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 17 Mar 2026 22:59:01 +0100 Subject: [PATCH 23/52] Statistiques => Observatoire #65 --- src/app.py | 4 ++-- src/pages/{statistiques.py => observatoire.py} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename src/pages/{statistiques.py => observatoire.py} (99%) diff --git a/src/app.py b/src/app.py index eb1931c..7a56058 100644 --- a/src/app.py +++ b/src/app.py @@ -53,7 +53,7 @@ def sitemap(): base_url = "https://decp.info" pages = [ "/", - "/statistiques", + "/observatoire", "/tableau", "/a-propos", ] @@ -160,7 +160,7 @@ navbar = dbc.Navbar( ) for page in page_registry.values() if page["name"] - in ["Recherche", "À propos", "Tableau", "Statistiques"] + in ["Recherche", "À propos", "Tableau", "Observatoire"] ], className="ms-auto", navbar=True, diff --git a/src/pages/statistiques.py b/src/pages/observatoire.py similarity index 99% rename from src/pages/statistiques.py rename to src/pages/observatoire.py index 9de6a29..eb726f7 100644 --- a/src/pages/statistiques.py +++ b/src/pages/observatoire.py @@ -14,12 +14,12 @@ from src.utils import ( meta_content, ) -name = "Statistiques" +name = "Observatoire" register_page( __name__, - path="/statistiques", - title="Statistiques | decp.info", + path="/observatoire", + title="Observatoire | decp.info", name=name, description="Visualisez l'état de la publication des données essentielles des marchés publics en France.", image_url=meta_content["image_url"], From 2f90a754ab32d4da701e76a55ef4e773ce82e855 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 13:38:38 +0100 Subject: [PATCH 24/52] =?UTF-8?q?Style=20inputs,=20ajout=20donut=20type=20?= =?UTF-8?q?march=C3=A9=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/assets/css/style.css | 10 ++ src/figures.py | 95 ++++++------- src/pages/observatoire.py | 274 +++++++++++++++++++++++++++++--------- 3 files changed, 271 insertions(+), 108 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 9705af4..12dd156 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -197,6 +197,16 @@ p.version > a { background-color: rgba(255, 240, 240, 0.4); } +#filters .col > * { + margin-bottom: 6px; +} + +#filters input[type="text"] { + border: 1px #ccc solid; + border-radius: 3px; + padding-left: 8px; +} + /* --- Tables (Dash & Custom) --- */ /* Table Menu (Exports etc) */ diff --git a/src/figures.py b/src/figures.py index c32f865..6f737cc 100644 --- a/src/figures.py +++ b/src/figures.py @@ -52,19 +52,18 @@ def get_yearly_statistics(statistics, today_str) -> html.Div: return html.Div(children=table, className="marches_table") -def get_barchart_sources(df_source: pl.DataFrame, type_date: str): - lf = df_source.lazy() +def get_barchart_sources(lff: pl.LazyFrame, type_date: str): labels = { "dateNotification": "notification", "datePublicationDonnees": "publication des données", } - lf = lf.select("uid", type_date, "sourceDataset") + lff = lff.select("uid", type_date, "sourceDataset") - lf = lf.unique("uid") + lff = lff.unique("uid") # Rassemblement des datasets Atexo pour ne pas surcharger le graphique - lf = lf.with_columns( + lff = lff.with_columns( pl.when(pl.col("sourceDataset").str.starts_with("atexo")) .then(pl.lit("plateformes atexo")) .otherwise(pl.col("sourceDataset")) @@ -72,34 +71,34 @@ def get_barchart_sources(df_source: pl.DataFrame, type_date: str): ) # Rassemblement des datasets AWS pour ne pas surcharger le graphique - lf = lf.with_columns( + lff = lff.with_columns( pl.when(pl.col("sourceDataset").str.contains(r"aws|marches\-publics.info")) .then(pl.lit("aws")) .otherwise(pl.col("sourceDataset")) .alias("sourceDataset") ) - lf = lf.with_columns(pl.col(type_date).dt.year().alias("annee")) - lf = lf.filter( + lff = lff.with_columns(pl.col(type_date).dt.year().alias("annee")) + lff = lff.filter( pl.col(type_date).is_not_null() & pl.col("annee").is_between(2019, 2025) ) - lf = lf.with_columns(pl.col(type_date).cast(pl.String).str.head(7)) - lf = ( - lf.group_by([type_date, "sourceDataset"]) + lff = lff.with_columns(pl.col(type_date).cast(pl.String).str.head(7)) + lff = ( + lff.group_by([type_date, "sourceDataset"]) .len() .sort(by=[type_date, "len"], descending=True) ) - # lf = lf.with_columns( + # lff = lff.with_columns( # pl.when(pl.col("sourceDataset").is_null()).then( # pl.lit("Source inconnue")).alias("sourceDataset") # ) - lf = lf.sort(by=["sourceDataset"], descending=False) - df: pl.DataFrame = lf.collect(engine="streaming") + lff = lff.sort(by=["sourceDataset"], descending=False) + dff: pl.DataFrame = lff.collect(engine="streaming") fig = px.bar( - df, + dff, x=type_date, y="len", color="sourceDataset", @@ -111,7 +110,9 @@ def get_barchart_sources(df_source: pl.DataFrame, type_date: str): }, ) - return fig + graph = dcc.Graph(figure=fig) + + return graph def get_sources_tables(source_path) -> html.Div: @@ -301,33 +302,24 @@ class DataTable(dash_table.DataTable): ) -def get_duplicate_matrix() -> html.Div: +def get_duplicate_matrix() -> dcc.Graph: """ Fonction développée avec l'aide de la LLM Euria d'Infomaniak. :return: """ - result_df = pl.read_parquet( + lff = pl.scan_parquet( "https://www.data.gouv.fr/api/1/datasets/r/a545bf6c-8b24-46ed-b49f-a32bf02eaffa" ).sort("sourceDataset") - result_df = result_df.select( - ["sourceDataset", "unique"] + sorted(result_df.columns[2:]) + lff = lff.select( + ["sourceDataset", "unique"] + sorted(lff.collect_schema().names()[2:]) ) - description = dcc.Markdown(""" - Ce graphique illustre les doublons de marchés publics entre sources, c'est-à-dire la proportion de marchés publiés par plus d'une source. Il s'appuie sur les identifiants `uid` qui sont pour chaque marché la concaténation du SIRET de l'acheteur et de l'identifiant interne du marché. - - **Comment lire ce graphique ?** - - On part des codes de sources de données en ordonnée. Ces jeux de données sont documentés dans [À propos](/a-propos#sources). - - La première colonne (**unique**) représente le pourcentage de marchés fournis par cette source qui sont uniquement disponibles dans cette source. Plus le rouge est foncé, plus important est le pourcentage. Donc, à l'inverse, plus le rouge est clair dans la première colonne, plus la source en ordonnée a des marchés en commun avec d'autres sources, et donc plus on trouvera sur la même ligne d'autres cases plus ou moins foncées qui indiqueront avec quelles autres sources cette source partage des marchés. - - Passez votre souris sur une case pour avoir les pourcentages exacts. À noter que ces statistiques sont produites avant le dédoublonnement qui a lieu avant la publication en Open Data et sur ce site.""") + dff = lff.collect() # Extract data - z_data = result_df.select(pl.all().exclude("sourceDataset")).fill_null(0).to_numpy() - x_labels = result_df.columns[1:] # columns after "sourceDataset" - y_labels = result_df["sourceDataset"].to_list() + z_data = dff.select(pl.all().exclude("sourceDataset")).fill_null(0).to_numpy() + x_labels = dff.columns[1:] # columns after "sourceDataset" + y_labels = dff["sourceDataset"].to_list() # Create heatmap fig = go.Figure( @@ -345,7 +337,7 @@ def get_duplicate_matrix() -> html.Div: hoverongaps=False, showscale=True, hovertemplate=( - "%{z:.0%} des marchés de %{y} sont également présents dans %{x}" + "%{z:.0%} des marchés présents dans %{y} sont également présents dans %{x}" ), ) ) @@ -364,13 +356,7 @@ def get_duplicate_matrix() -> html.Div: margin=dict(l=100, r=50, t=80, b=100), # Add margin for labels ) - return html.Div( - children=[ - html.H3("Doublons de marchés entre les sources"), - description, - dcc.Graph(figure=fig), - ] - ) + return dcc.Graph(figure=fig) def get_geographic_maps(dff: pl.DataFrame) -> list | None: @@ -624,20 +610,39 @@ def make_card( return card -def make_donut(lff: pl.LazyFrame, names_col): +def make_donut( + lff: pl.LazyFrame, + names_col, + per_uid: bool, + nulls="?", +): title = data_schema[names_col]["title"] lff = lff.rename({names_col: title}) lff = lff.select("uid", title) + + if per_uid: + lff = lff.group_by("uid").first() + lff = lff.group_by(title).len("Nombre") - lff = lff.with_columns(pl.col(title).replace(None, pl.lit("?"))) + lff = lff.with_columns(pl.col(title).replace(None, pl.lit(nulls))) + dff = lff.collect(engine="streaming") + dff = dff.with_columns( + pl.col("Nombre") + .map_elements(format_number, return_dtype=pl.String) + .alias("Nombre_fmt") + ) fig = px.pie( - lff.collect(engine="streaming"), + dff, values="Nombre", names=title, hole=0.4, color_discrete_sequence=px.colors.qualitative.Safe, + custom_data=["Nombre_fmt"], + ) + fig = fig.update_traces( + texttemplate="%{label}
%{percent}", + hovertemplate="%{label}
%{customdata[0]}", ) - fig = fig.update_traces(texttemplate="%{label}
%{percent}") fig = fig.update_layout(showlegend=False, font=dict(size=14)) graph = dcc.Graph(figure=fig) return graph diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index eb726f7..2b2cee8 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -3,9 +3,15 @@ from datetime import datetime, timedelta import dash_bootstrap_components as dbc import polars as pl import polars.selectors as cs -from dash import Input, Output, callback, dcc, html, register_page +from dash import ALL, Input, Output, callback, ctx, dcc, html, register_page -from src.figures import get_geographic_maps, make_card, make_donut +from src.figures import ( + get_barchart_sources, + get_duplicate_matrix, + get_geographic_maps, + make_card, + make_donut, +) from src.utils import ( departements, df, @@ -38,6 +44,29 @@ for code, obj in departements.items(): layout = [ dcc.Store(id="dashboard-filters"), dcc.Location(id="dashboard_url"), + dbc.Modal( + [ + dbc.ModalHeader(dbc.ModalTitle("Montants")), + dbc.ModalBody( + [ + dcc.Markdown( + """ +Les données saisies et publiées par les acheteurs comportent de nombreux montants farfelus qui sabotent les statistiques, au lieu de montants estimés avec rigueur. On parle de montant atteignant parfois les millions de milliards. Certains réutilisateurs mettent de côté ces marchés ou bien modifient les montants selon des règles fatalement arbitraires. J'ai fait le choix de ne quasiment pas modifier les données* afin de visibiliser le problème. + +Alors, on fait comment ? + +\* Les montants composés de plus de 11 chiffres, sans les décimales, [sont ramenés](https://github.com/ColinMaudry/decp-processing/blob/main/src/tasks/clean.py#L63-L71) à 12 311 111 111, un nombre qui reste très élevé et qui est facilement reconnaissable. +""" + ), + ] + ), + dbc.ModalFooter( + dbc.Button("Fermer", id="montant-modal-close", className="ms-auto") + ), + ], + id="montant-modal", + is_open=False, + ), html.Div( className="container-fluid", children=[ @@ -56,66 +85,109 @@ layout = [ children=[ html.H5("Période d'attribution"), dbc.Row( - dcc.Dropdown( - id="dashboard_year", - options=options_years, - placeholder="12 derniers mois", + dbc.Col( + dcc.Dropdown( + id="dashboard_year", + options=options_years, + placeholder="12 derniers mois", + ), ), ), html.H5("Acheteur"), dbc.Row( - dcc.Dropdown( - id="dashboard_acheteur_categorie", - options=get_enum_values_as_dict( - "acheteur_categorie" + dbc.Col( + dcc.Input( + id="dashboard_acheteur_id", + placeholder="SIRET", + style={"width": "100%"}, ), - placeholder="Catégorie", ), ), dbc.Row( - dcc.Dropdown( - id="dashboard_acheteur_departement_code", - searchable=True, - multi=True, - placeholder="Code département acheteur", - options=options_departements, + dbc.Col( + dcc.Dropdown( + id="dashboard_acheteur_categorie", + options=get_enum_values_as_dict( + "acheteur_categorie" + ), + placeholder="Catégorie", + ) + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_acheteur_departement_code", + searchable=True, + multi=True, + placeholder="Département", + options=options_departements, + ), ), ), html.H5("Titulaire"), dbc.Row( - dcc.Dropdown( - id="dashboard_titulaire_categorie", - placeholder="Catégorie", - options=get_enum_values_as_dict( - "titulaire_categorie" + dbc.Col( + dcc.Input( + id="dashboard_titulaire_id", + placeholder="SIRET", + style={"width": "100%"}, + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_titulaire_categorie", + placeholder="Catégorie", + options=get_enum_values_as_dict( + "titulaire_categorie" + ), + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_titulaire_departement_code", + searchable=True, + multi=True, + placeholder="Département", + options=options_departements, ), ), ), html.H5("Marché"), dbc.Row( - dcc.Dropdown( - id="dashboard_marche_type", - placeholder="Type", - options=get_enum_values_as_dict("type"), - ), - ), - dbc.Row( - dcc.Dropdown( - id="dashboard_marche_considerationsSociales", - placeholder="Considérations sociales", - options=get_enum_values_as_dict( - "considerationsSociales" + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_type", + placeholder="Type", + options=get_enum_values_as_dict("type"), ), - multi=True, ), ), dbc.Row( - dcc.Dropdown( - id="dashboard_marche_considerationsEnvironnementales", - placeholder="Considérations environnementales", - multi=True, - options=get_enum_values_as_dict( - "considerationsEnvironnementales" + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_considerationsSociales", + placeholder="Considérations sociales", + options=get_enum_values_as_dict( + "considerationsSociales" + ), + multi=True, + ), + ), + ), + dbc.Row( + dbc.Col( + dcc.Dropdown( + id="dashboard_marche_considerationsEnvironnementales", + placeholder="Considérations environnementales", + multi=True, + options=get_enum_values_as_dict( + "considerationsEnvironnementales" + ), ), ), ), @@ -140,21 +212,27 @@ layout = [ @callback( Output("cards", "children"), Input("dashboard_year", "value"), + Input("dashboard_acheteur_id", "value"), Input("dashboard_acheteur_categorie", "value"), Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_titulaire_id", "value"), Input("dashboard_titulaire_categorie", "value"), + Input("dashboard_titulaire_departement_code", "value"), Input("dashboard_marche_type", "value"), Input("dashboard_marche_considerationsSociales", "value"), Input("dashboard_marche_considerationsEnvironnementales", "value"), ) def udpate_dashboard_cards( dashboard_year, + dashboard_acheteur_id, dashboard_acheteur_categorie, dashboard_acheteur_departement_code, + dashboard_titulaire_id, dashboard_titulaire_categorie, + dashboard_titulaire_departement_code, dashboard_marche_type, - dashboard_marche_considerationsSociales, - dashboard_marche_considerationsEnvironnementales, + dashboard_marche_considerations_sociales, + dashboard_marche_considerations_environnementales, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -165,6 +243,8 @@ def udpate_dashboard_cards( "montant", "considerationsSociales", "considerationsEnvironnementales", + "sourceDataset", + "type", ) # Application des filtres @@ -180,40 +260,57 @@ def udpate_dashboard_cards( ## Acheteur - if dashboard_acheteur_categorie: - lff = lff.filter(pl.col("acheteur_categorie") == dashboard_acheteur_categorie) - - if dashboard_acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in( - dashboard_acheteur_departement_code + if dashboard_acheteur_id: + lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) + else: + if dashboard_acheteur_categorie: + lff = lff.filter( + pl.col("acheteur_categorie") == dashboard_acheteur_categorie + ) + + if dashboard_acheteur_departement_code: + lff = lff.filter( + pl.col("acheteur_departement_code").is_in( + dashboard_acheteur_departement_code + ) ) - ) ## Titulaire - if dashboard_titulaire_categorie: - lff = lff.filter(pl.col("titulaire_categorie") == dashboard_titulaire_categorie) + if dashboard_titulaire_id: + lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) + else: + if dashboard_titulaire_categorie: + lff = lff.filter( + pl.col("titulaire_categorie") == dashboard_titulaire_categorie + ) + + if dashboard_titulaire_departement_code: + lff = lff.filter( + pl.col("titulaire_departement_code").is_in( + dashboard_titulaire_departement_code + ) + ) ## Marché if dashboard_marche_type: lff = lff.filter(pl.col("type") == dashboard_marche_type) - if dashboard_marche_considerationsSociales: + if dashboard_marche_considerations_sociales: lff = lff.filter( pl.col("considerationsSociales") .str.split(", ") - .list.set_intersection(dashboard_marche_considerationsSociales) + .list.set_intersection(dashboard_marche_considerations_sociales) .list.len() > 0 ) - if dashboard_marche_considerationsEnvironnementales: + if dashboard_marche_considerations_environnementales: lff = lff.filter( pl.col("considerationsEnvironnementales") .str.split(", ") - .list.set_intersection(dashboard_marche_considerationsEnvironnementales) + .list.set_intersection(dashboard_marche_considerations_environnementales) .list.len() > 0 ) @@ -228,7 +325,8 @@ def udpate_dashboard_cards( df_per_uid = ( dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first()) ) - total_montant = df_per_uid.select(pl.col("montant").sum()).item() + + total_montant = int(df_per_uid.select(pl.col("montant").sum()).item()) nb_marches = df_per_uid.height cards = [] @@ -241,19 +339,69 @@ def udpate_dashboard_cards( html.P( ["Nombre de titulaires : ", html.Strong(str(format_number(nb_titulaires)))] ), - html.P(["Montant total : ", html.Strong(format_number(total_montant) + " €")]), + html.P( + [ + "Montant total (", + html.Span( + "?", + id={"type": "modal-trigger", "index": "montant"}, + style={"cursor": "pointer", "textDecoration": "underline dotted"}, + ), + ") : ", + html.Strong(format_number(total_montant) + " €"), + ] + ), ] cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) - donut_acheteur_categorie = make_donut(lff, "acheteur_categorie") - cards.append(make_card(title="Catégorie d'acheteur", fig=donut_acheteur_categorie)) + donut_acheteur_categorie = make_donut( + lff, "acheteur_categorie", nulls="Autres", per_uid=True + ) + cards.append( + make_card( + title="Catégorie d'acheteur", fig=donut_acheteur_categorie, lg=12, xl=8 + ) + ) - donut_titulaire_categorie = make_donut(lff, "titulaire_categorie") + donut_titulaire_categorie = make_donut( + lff, "titulaire_categorie", per_uid=False, nulls="?" + ) cards.append( make_card(title="Catégorie d'entreprise", fig=donut_titulaire_categorie) ) + donut_marche_type = make_donut(lff, "type", per_uid=True, nulls="?") + cards.append(make_card(title="Type d'achat", fig=donut_marche_type)) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) - return dbc.Row(children=cards + geographic_maps) + other_cards = [] + + sources_barchart = get_barchart_sources(lff, type_date="dateNotification") + other_cards.append( + make_card(title="Sources de données", fig=sources_barchart, lg=12, xl=8) + ) + + duplicate_matrix = get_duplicate_matrix() + other_cards.append( + make_card( + title="Matrice de doublons entre sources de données", + subtitle="Ce graphique illustre les doublons de marchés publics entre sources, c'est-à-dire la proportion de marchés publiés par plus d'une source.", + fig=duplicate_matrix, + lg=12, + xl=8, + ) + ) + + return dbc.Row(children=cards + geographic_maps + other_cards) + + +@callback( + Output("montant-modal", "is_open"), + Input({"type": "modal-trigger", "index": ALL}, "n_clicks"), + Input("montant-modal-close", "n_clicks"), + prevent_initial_call=True, +) +def toggle_montant_modal(n_triggers, _close): + return isinstance(ctx.triggered_id, dict) and any(n_triggers) From 79a06f996e46eed25ef1b4ad10a1112826d82eec Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 13:52:29 +0100 Subject: [PATCH 25/52] =?UTF-8?q?Bouton=20de=20t=C3=A9l=C3=A9chargement=20?= =?UTF-8?q?des=20donn=C3=A9es=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/figures.py | 1 - src/pages/observatoire.py | 224 ++++++++++++++++++++++++++------------ 2 files changed, 152 insertions(+), 73 deletions(-) diff --git a/src/figures.py b/src/figures.py index 6f737cc..8337f9c 100644 --- a/src/figures.py +++ b/src/figures.py @@ -102,7 +102,6 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): x=type_date, y="len", color="sourceDataset", - title=f"Nombre de marchés attribués par date de {labels[type_date]} et source de données", labels={ "len": "Nombre de marchés", type_date: f"Mois de {labels[type_date]}", diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 2b2cee8..d48518c 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta import dash_bootstrap_components as dbc import polars as pl import polars.selectors as cs -from dash import ALL, Input, Output, callback, ctx, dcc, html, register_page +from dash import ALL, Input, Output, State, callback, ctx, dcc, html, register_page from src.figures import ( get_barchart_sources, @@ -41,6 +41,70 @@ for code, obj in departements.items(): options_departements[code] = f"{obj['departement']} ({code})" +def _apply_filters( + lff: pl.LazyFrame, + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, +) -> pl.LazyFrame: + if year: + lff = lff.filter(pl.col("dateNotification").dt.year() == int(year)) + else: + lff = lff.filter( + pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) + ) + + if acheteur_id: + lff = lff.filter(pl.col("acheteur_id").str.contains(acheteur_id)) + else: + if acheteur_categorie: + lff = lff.filter(pl.col("acheteur_categorie") == acheteur_categorie) + if acheteur_departement_code: + lff = lff.filter( + pl.col("acheteur_departement_code").is_in(acheteur_departement_code) + ) + + if titulaire_id: + lff = lff.filter(pl.col("titulaire_id").str.contains(titulaire_id)) + else: + if titulaire_categorie: + lff = lff.filter(pl.col("titulaire_categorie") == titulaire_categorie) + if titulaire_departement_code: + lff = lff.filter( + pl.col("titulaire_departement_code").is_in(titulaire_departement_code) + ) + + if marche_type: + lff = lff.filter(pl.col("type") == marche_type) + + if considerations_sociales: + lff = lff.filter( + pl.col("considerationsSociales") + .str.split(", ") + .list.set_intersection(considerations_sociales) + .list.len() + > 0 + ) + + if considerations_environnementales: + lff = lff.filter( + pl.col("considerationsEnvironnementales") + .str.split(", ") + .list.set_intersection(considerations_environnementales) + .list.len() + > 0 + ) + + return lff + + layout = [ dcc.Store(id="dashboard-filters"), dcc.Location(id="dashboard_url"), @@ -55,7 +119,7 @@ Les données saisies et publiées par les acheteurs comportent de nombreux monta Alors, on fait comment ? -\* Les montants composés de plus de 11 chiffres, sans les décimales, [sont ramenés](https://github.com/ColinMaudry/decp-processing/blob/main/src/tasks/clean.py#L63-L71) à 12 311 111 111, un nombre qui reste très élevé et qui est facilement reconnaissable. +\\* Les montants composés de plus de 11 chiffres, sans les décimales, [sont ramenés](https://github.com/ColinMaudry/decp-processing/blob/main/src/tasks/clean.py#L63-L71) à 12 311 111 111, un nombre qui reste très élevé et qui est facilement reconnaissable. """ ), ] @@ -191,6 +255,13 @@ Alors, on fait comment ? ), ), ), + dcc.Download(id="download-observatoire"), + dbc.Button( + "Télécharger au format Excel", + id="btn-download-observatoire", + disabled=True, + className="mt-2", + ), ], ), dbc.Col( @@ -211,6 +282,8 @@ Alors, on fait comment ? @callback( Output("cards", "children"), + Output("btn-download-observatoire", "disabled"), + Output("btn-download-observatoire", "children"), Input("dashboard_year", "value"), Input("dashboard_acheteur_id", "value"), Input("dashboard_acheteur_categorie", "value"), @@ -246,74 +319,19 @@ def udpate_dashboard_cards( "sourceDataset", "type", ) - - # Application des filtres - - ## Période - - if dashboard_year: - lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) - else: - lff = lff.filter( - pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) - ) - - ## Acheteur - - if dashboard_acheteur_id: - lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) - else: - if dashboard_acheteur_categorie: - lff = lff.filter( - pl.col("acheteur_categorie") == dashboard_acheteur_categorie - ) - - if dashboard_acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in( - dashboard_acheteur_departement_code - ) - ) - - ## Titulaire - - if dashboard_titulaire_id: - lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) - else: - if dashboard_titulaire_categorie: - lff = lff.filter( - pl.col("titulaire_categorie") == dashboard_titulaire_categorie - ) - - if dashboard_titulaire_departement_code: - lff = lff.filter( - pl.col("titulaire_departement_code").is_in( - dashboard_titulaire_departement_code - ) - ) - - ## Marché - - if dashboard_marche_type: - lff = lff.filter(pl.col("type") == dashboard_marche_type) - - if dashboard_marche_considerations_sociales: - lff = lff.filter( - pl.col("considerationsSociales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_sociales) - .list.len() - > 0 - ) - - if dashboard_marche_considerations_environnementales: - lff = lff.filter( - pl.col("considerationsEnvironnementales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_environnementales) - .list.len() - > 0 - ) + lff = _apply_filters( + lff, + dashboard_year, + dashboard_acheteur_id, + dashboard_acheteur_categorie, + dashboard_acheteur_departement_code, + dashboard_titulaire_id, + dashboard_titulaire_categorie, + dashboard_titulaire_departement_code, + dashboard_marche_type, + dashboard_marche_considerations_sociales, + dashboard_marche_considerations_environnementales, + ) # Génération des métriques dff = lff.collect(engine="streaming") @@ -329,6 +347,13 @@ def udpate_dashboard_cards( total_montant = int(df_per_uid.select(pl.col("montant").sum()).item()) nb_marches = df_per_uid.height + if nb_marches == 0: + dl_disabled, dl_text = True, "Pas de données à télécharger" + elif nb_marches > 65000: + dl_disabled, dl_text = True, "Téléchargement désactivé au-delà de 65 000 lignes" + else: + dl_disabled, dl_text = False, "Télécharger au format Excel" + cards = [] card_basic_counts = [ @@ -380,7 +405,13 @@ def udpate_dashboard_cards( sources_barchart = get_barchart_sources(lff, type_date="dateNotification") other_cards.append( - make_card(title="Sources de données", fig=sources_barchart, lg=12, xl=8) + make_card( + title="Sources de données", + subtitle="Nombre de marchés attribués par mois de notification et source de données", + fig=sources_barchart, + lg=12, + xl=8, + ) ) duplicate_matrix = get_duplicate_matrix() @@ -394,7 +425,56 @@ def udpate_dashboard_cards( ) ) - return dbc.Row(children=cards + geographic_maps + other_cards) + return dbc.Row(children=cards + geographic_maps + other_cards), dl_disabled, dl_text + + +@callback( + Output("download-observatoire", "data"), + Input("btn-download-observatoire", "n_clicks"), + State("dashboard_year", "value"), + State("dashboard_acheteur_id", "value"), + State("dashboard_acheteur_categorie", "value"), + State("dashboard_acheteur_departement_code", "value"), + State("dashboard_titulaire_id", "value"), + State("dashboard_titulaire_categorie", "value"), + State("dashboard_titulaire_departement_code", "value"), + State("dashboard_marche_type", "value"), + State("dashboard_marche_considerationsSociales", "value"), + State("dashboard_marche_considerationsEnvironnementales", "value"), + prevent_initial_call=True, +) +def download_observatoire( + _n_clicks, + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, +): + lff = _apply_filters( + df.lazy(), + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, + ) + + def to_bytes(buffer): + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") + + date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes(to_bytes, filename=f"decp_observatoire_{date}.xlsx") @callback( From 3a70bbd9eaa3201b99de0c7a4836ed9691c1cd05 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:24:11 +0100 Subject: [PATCH 26/52] Ajout du spec : lien observatoire depuis recherche/tableau #65 Co-Authored-By: Claude Opus 4.6 --- ...18-observatoire-link-from-search-design.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md diff --git a/docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md b/docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md new file mode 100644 index 0000000..8843c0c --- /dev/null +++ b/docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md @@ -0,0 +1,78 @@ +# Observatoire Link from Search & Tableau Results + +## Problem + +Users searching for an organization (acheteur or titulaire) on the search page or browsing the tableau cannot jump directly to the observatoire page filtered for that organization. They must manually navigate and re-enter the identifier. + +## Solution + +Extend `add_links()` in `src/utils.py` to append an observatoire link (📊 emoji) to `_nom` columns, and add bidirectional URL parameter sync to the observatoire page. + +## Changes + +### 1. `src/utils.py` — `add_links()` modification + +The existing `add_links()` loop iterates over `["uid", "acheteur_nom", "titulaire_nom", "acheteur_id", "titulaire_id"]`. The `if col.startswith("acheteur_")` and `if col.startswith("titulaire_")` blocks match both `_nom` and `_id` columns. The observatoire link must only be appended to `_nom` columns, so it must be gated on `col == "acheteur_nom"` or `col == "titulaire_nom"` explicitly. + +For `acheteur_nom`, append an observatoire link after the existing detail page link: + +``` +Before: Ville de Paris +After: Ville de Paris 📊 +``` + +For `titulaire_nom`, same pattern but only when the existing `typeIdentifiant` guard passes (SIRET or null): + +``` +Before: Entreprise X +After: Entreprise X 📊 +``` + +The identifier used in the observatoire link (`acheteur_id` / `titulaire_id`) is the same `pl.col("acheteur_id")` / `pl.col("titulaire_id")` column value already used for the detail page link. + +The `_id` and `uid` columns are unchanged. + +### 2. `src/pages/observatoire.py` — URL parameter handling + +#### Callback A: URL → Inputs (page load) + +- Trigger: `Input("dashboard_url", "search")` +- Outputs: `Output("dashboard_acheteur_id", "value")`, `Output("dashboard_titulaire_id", "value")`, `Output("dashboard_url", "search")` (to clear it) +- `prevent_initial_call=False` (must fire on page load to read URL params) +- If `search` is empty or None: return `no_update` for all outputs +- Otherwise: parse query params with `urllib.parse.parse_qs` +- Set `dashboard_acheteur_id` from `?acheteur_id=` param, or `no_update` if absent +- Set `dashboard_titulaire_id` from `?titulaire_id=` param, or `no_update` if absent +- Return `""` for `dashboard_url.search` to clear the URL and prevent re-triggering +- No validation of param values — consistent with existing input handling in the observatoire callbacks + +#### Callback B: Inputs → shareable URL + +- Trigger: `Input("dashboard_acheteur_id", "value")`, `Input("dashboard_titulaire_id", "value")` +- State: `State("dashboard_url", "href")` for base URL +- `prevent_initial_call=True` (avoid generating URL on initial empty state) +- Build query string with `urllib.parse.urlencode`, omitting empty values +- Write full URL to a new `share-url` input component +- Render a `dcc.Clipboard` + share button (same pattern as tableau.py) + +#### Callback chain + +When navigating from search with `?acheteur_id=123`: Callback A fires on page load, sets input values, clears URL search. The input value changes then trigger both the existing `udpate_dashboard_cards` callback and Callback B. Dash handles this chaining deterministically — no race condition. + +#### Layout additions + +- A `dcc.Input(id="share-url", ...)` (hidden or read-only) to hold the shareable URL +- A `dcc.Clipboard` share/copy button near the filters + +### 3. Reuse of existing `dcc.Location` + +The existing `dcc.Location(id="dashboard_url")` component is reused — no new Location component needed. + +## Future extension + +The bidirectional URL sync pattern is designed to extend to all observatoire filters (year, categories, departments, market type, etc.) by adding more params to both callbacks. + +## Files touched + +- `src/utils.py` — modify `add_links()` +- `src/pages/observatoire.py` — add 2 callbacks, add share-url + clipboard to layout From d1a876ba9cfd17a571e071ed6bb54580afd1cc10 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:35:58 +0100 Subject: [PATCH 27/52] =?UTF-8?q?Plan=20d'impl=C3=A9mentation=20:=20lien?= =?UTF-8?q?=20observatoire=20depuis=20recherche/tableau=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- .../plans/2026-03-18-observatoire-link.md | 473 ++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-18-observatoire-link.md diff --git a/docs/superpowers/plans/2026-03-18-observatoire-link.md b/docs/superpowers/plans/2026-03-18-observatoire-link.md new file mode 100644 index 0000000..f81a2d4 --- /dev/null +++ b/docs/superpowers/plans/2026-03-18-observatoire-link.md @@ -0,0 +1,473 @@ +# Observatoire Link from Search & Tableau Results — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let users jump from search/tableau results to the observatoire page, pre-filtered for a given organization, via a 📊 link in the `_nom` columns. + +**Architecture:** Modify `add_links()` in `src/utils.py` to append an observatoire link to `_nom` columns. Add two callbacks to `src/pages/observatoire.py` for bidirectional URL ↔ filter sync using the existing `dcc.Location(id="dashboard_url")`. Add a share URL input and clipboard button to the observatoire layout. + +**Tech Stack:** Dash 3.4, Polars, `urllib.parse`, `dcc.Location`, `dcc.Clipboard` + +**Spec:** `docs/superpowers/specs/2026-03-18-observatoire-link-from-search-design.md` + +--- + +### Task 1: Add observatoire link to `acheteur_nom` in `add_links()` + +**Files:** + +- Modify: `src/utils.py:82-91` (the `acheteur_` block inside `add_links()`) +- Test: `tests/test_main.py` + +**Context:** The `add_links()` function loops over column names. The `if col.startswith("acheteur_")` block (lines 82-91) currently wraps both `acheteur_nom` and `acheteur_id` in a detail page link. We must only append the observatoire link when `col == "acheteur_nom"`. + +- [ ] **Step 1: Write a unit test for the observatoire link in acheteur_nom** + +In `tests/test_main.py`, add a test that calls `add_links()` on a minimal DataFrame and checks the `acheteur_nom` column contains both the detail link and the observatoire link, while `acheteur_id` does NOT contain the observatoire link. + +```python +def test_004_add_links_observatoire_acheteur(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "acheteur_id": ["a1"], + "acheteur_nom": ["ACHETEUR 1"], + } + ) + result = add_links(dff) + nom_value = result["acheteur_nom"][0] + id_value = result["acheteur_id"][0] + + # acheteur_nom should contain detail link + observatoire link + assert "/acheteurs/a1" in nom_value + assert "ACHETEUR 1" in nom_value + assert '/observatoire?acheteur_id=a1' in nom_value + assert "📊" in nom_value + + # acheteur_id should NOT contain observatoire link + assert "/observatoire" not in id_value +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_004_add_links_observatoire_acheteur -v` +Expected: FAIL — `'/observatoire?acheteur_id=a1'` not found in the output string. + +- [ ] **Step 3: Implement the observatoire link for acheteur_nom** + +In `src/utils.py`, modify the `if col.startswith("acheteur_")` block (lines 82-91). Gate the observatoire link append on `col == "acheteur_nom"`: + +```python + if col.startswith("acheteur_"): + detail_link = ( + '' + + pl.col(col) + + "" + ) + if col == "acheteur_nom": + detail_link = ( + detail_link + + ' 📊' + ) + dff = dff.with_columns(detail_link.alias(col)) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_004_add_links_observatoire_acheteur -v` +Expected: PASS + +- [ ] **Step 5: Update `test_001` to account for the new emoji in cell text** + +The existing `test_001` asserts `result_table.find_element(...).text == name` for `acheteur_nom`. The cell text now includes "📊" from the observatoire link. Update the assertion in `tests/test_main.py` to use `startswith` instead of exact match: + +```python + assert result_table.find_element( + by=By.CSS_SELECTOR, value=f'td[data-dash-column="{org_type}_nom"]' + ).text.startswith( + name + ), f"The search result should have the right {org_type} name" +``` + +- [ ] **Step 6: Run `test_001` to verify it still passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_001_logo_and_search -v` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/utils.py tests/test_main.py +git commit -m "Ajout du lien observatoire dans acheteur_nom via add_links() #65" +``` + +--- + +### Task 2: Add observatoire link to `titulaire_nom` in `add_links()` + +**Files:** + +- Modify: `src/utils.py:64-81` (the `titulaire_` block inside `add_links()`) +- Test: `tests/test_main.py` + +**Context:** The `titulaire_` block (lines 64-81) uses a `pl.when().then().otherwise()` pattern because it guards on `titulaire_typeIdentifiant` being SIRET or null. The observatoire link must be appended inside the `.then()` branch, and only when `col == "titulaire_nom"`. Note: this block requires `titulaire_typeIdentifiant` to be present in the DataFrame. + +- [ ] **Step 1: Write a unit test for the observatoire link in titulaire_nom** + +```python +def test_005_add_links_observatoire_titulaire(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "titulaire_id": ["t1"], + "titulaire_nom": ["TITULAIRE 1"], + "titulaire_typeIdentifiant": ["SIRET"], + } + ) + result = add_links(dff) + nom_value = result["titulaire_nom"][0] + id_value = result["titulaire_id"][0] + + # titulaire_nom should contain detail link + observatoire link + assert "/titulaires/t1" in nom_value + assert "TITULAIRE 1" in nom_value + assert '/observatoire?titulaire_id=t1' in nom_value + assert "📊" in nom_value + + # titulaire_id should NOT contain observatoire link + assert "/observatoire" not in id_value +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_005_add_links_observatoire_titulaire -v` +Expected: FAIL — `'/observatoire?titulaire_id=t1'` not found. + +- [ ] **Step 3: Implement the observatoire link for titulaire_nom** + +In `src/utils.py`, modify the `if col.startswith("titulaire_")` block (lines 64-81). The `.then()` branch must build the link differently when `col == "titulaire_nom"`: + +```python + if col.startswith("titulaire_"): + detail_link = ( + '' + + pl.col(col) + + "" + ) + if col == "titulaire_nom": + detail_link = ( + detail_link + + ' 📊' + ) + dff = dff.with_columns( + pl.when( + pl.Expr.or_( + pl.col("titulaire_typeIdentifiant").is_null(), + pl.col("titulaire_typeIdentifiant") == "SIRET", + ) + ) + .then(detail_link) + .otherwise(pl.col(col)) + .alias(col) + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_005_add_links_observatoire_titulaire -v` +Expected: PASS + +- [ ] **Step 5: Run all tests so far to check for regressions** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_004_add_links_observatoire_acheteur tests/test_main.py::test_005_add_links_observatoire_titulaire -v` +Expected: both PASS + +- [ ] **Step 6: Commit** + +```bash +git add src/utils.py tests/test_main.py +git commit -m "Ajout du lien observatoire dans titulaire_nom via add_links() #65" +``` + +--- + +### Task 3: Observatoire Callback A — URL → Inputs (page load) + +**Files:** + +- Modify: `src/pages/observatoire.py` (add import + new callback after line 281) +- Test: `tests/test_main.py` + +**Context:** The existing `dcc.Location(id="dashboard_url")` is in the observatoire layout. A new callback reads `dashboard_url.search` on page load, parses query params, and sets `dashboard_acheteur_id.value` and/or `dashboard_titulaire_id.value`. It also clears `dashboard_url.search` to `""` to prevent re-triggering. Two imports must be added: `import urllib.parse` at the top of the file, and `no_update` to the existing `from dash import ...` line (currently: `from dash import ALL, Input, Output, State, callback, ctx, dcc, html, register_page` — add `no_update` to this). + +- [ ] **Step 1: Write a Selenium test for URL → Input sync** + +This test navigates to `/observatoire?acheteur_id=a1` and verifies the SIRET input gets populated. + +```python +def test_006_observatoire_url_to_input(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + dash_duo.wait_for_text_to_equal( + "#dashboard_acheteur_id", "", timeout=4 + ) # Wait for callback + import time + time.sleep(1) # Allow callback chain to complete + + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated from URL param" + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_006_observatoire_url_to_input -v` +Expected: FAIL — the input value is empty because no callback reads URL params yet. + +- [ ] **Step 3: Implement Callback A** + +Add `import urllib.parse` to the imports at the top of `src/pages/observatoire.py` (after line 1). Also add `no_update` to the existing dash import line: + +```python +from dash import ALL, Input, Output, State, callback, ctx, dcc, html, no_update, register_page +``` + +Add the callback after the `layout` list ends, before existing callbacks: + +```python +@callback( + Output("dashboard_acheteur_id", "value"), + Output("dashboard_titulaire_id", "value"), + Output("dashboard_url", "search"), + Input("dashboard_url", "search"), +) +def restore_filters_from_url(search): + if not search: + return no_update, no_update, no_update + + params = urllib.parse.parse_qs(search.lstrip("?")) + + acheteur_id = params.get("acheteur_id", [None])[0] or no_update + titulaire_id = params.get("titulaire_id", [None])[0] or no_update + + return acheteur_id, titulaire_id, "" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_006_observatoire_url_to_input -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/pages/observatoire.py tests/test_main.py +git commit -m "Callback URL → filtres sur la page observatoire #65" +``` + +--- + +### Task 4: Observatoire Callback B — Inputs → shareable URL + layout + +**Files:** + +- Modify: `src/pages/observatoire.py` (add layout components + new callback) +- Test: `tests/test_main.py` + +**Context:** Following the tableau.py pattern (lines 237-238 for layout, lines 399-450 for callback), add a hidden `share-url` input and a `copy-container` div to the observatoire layout. The callback listens to the ID inputs and builds a shareable URL. Component IDs must be unique across the app, so use `observatoire-share-url` and `observatoire-copy-container` to avoid collisions with tableau's `share-url` and `copy-container`. + +- [ ] **Step 1: Write a test for the shareable URL generation** + +```python +def test_007_observatoire_share_url(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#observatoire-share-url", timeout=4) + + import time + time.sleep(1) # Allow callback chain to complete + + share_url_input = dash_duo.find_element("#observatoire-share-url") + share_url_value = share_url_input.get_attribute("value") + + assert "acheteur_id=a1" in share_url_value, ( + f"Share URL should contain acheteur_id param, got: {share_url_value}" + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_007_observatoire_share_url -v` +Expected: FAIL — `#observatoire-share-url` element does not exist yet. + +- [ ] **Step 3: Add layout components to observatoire** + +In `src/pages/observatoire.py`, add the share URL input and copy container inside the filters column (after the download button, before the closing `]` of the `id="filters"` children list, around line 264): + +```python + dcc.Input( + id="observatoire-share-url", + readOnly=True, + style={"display": "none"}, + ), + html.Div(id="observatoire-copy-container"), +``` + +- [ ] **Step 4: Implement Callback B** + +Add after Callback A in `src/pages/observatoire.py`: + +```python +@callback( + Output("observatoire-share-url", "value"), + Output("observatoire-copy-container", "children"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_titulaire_id", "value"), + State("dashboard_url", "href"), + prevent_initial_call=True, +) +def sync_observatoire_share_url(acheteur_id, titulaire_id, href): + if not href: + return no_update, no_update + + base_url = href.split("?")[0] + + params = {} + if acheteur_id: + params["acheteur_id"] = acheteur_id + if titulaire_id: + params["titulaire_id"] = titulaire_id + + query_string = urllib.parse.urlencode(params) + full_url = f"{base_url}?{query_string}" if query_string else base_url + + copy_button = dcc.Clipboard( + id="btn-copy-observatoire-url", + target_id="observatoire-share-url", + title="Copier l'URL de cette vue", + style={ + "display": "inline-block", + "fontSize": 20, + "verticalAlign": "top", + "cursor": "pointer", + }, + className="fa fa-link", + children=[ + dbc.Button( + "Partager", + className="btn btn-primary mt-2", + title="Copier l'adresse de cette vue filtrée pour la partager.", + ) + ], + ) + + return full_url, copy_button +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_007_observatoire_share_url -v` +Expected: PASS + +- [ ] **Step 6: Run all tests to check for regressions** + +Run: `source .venv/bin/activate && pytest tests/test_main.py -v` +Expected: all tests PASS + +- [ ] **Step 7: Commit** + +```bash +git add src/pages/observatoire.py tests/test_main.py +git commit -m "URL partageable pour la page observatoire #65" +``` + +--- + +### Task 5: End-to-end integration test + +**Files:** + +- Test: `tests/test_main.py` + +**Context:** Verify the full flow: search for an organization on the homepage, see the 📊 link in results, click it, arrive on the observatoire with the correct input populated. + +- [ ] **Step 1: Write end-to-end test** + +```python +def test_008_search_to_observatoire(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Search for an acheteur + search_bar = dash_duo.find_element("#search") + search_bar.send_keys("ACHETEUR 1") + search_bar.send_keys(Keys.ENTER) + + dash_duo.wait_for_element("#results_acheteur_datatable", timeout=2) + + # Find the observatoire link in acheteur_nom column + observatoire_link = dash_duo.find_element( + '#results_acheteur_datatable td[data-dash-column="acheteur_nom"] a[href*="observatoire"]' + ) + assert "📊" in observatoire_link.text + + # Click the observatoire link + observatoire_link.click() + + # Wait for observatoire page to load + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + import time + time.sleep(1) # Allow callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated after navigating from search" + ) +``` + +- [ ] **Step 2: Run end-to-end test** + +Run: `source .venv/bin/activate && pytest tests/test_main.py::test_008_search_to_observatoire -v` +Expected: PASS + +- [ ] **Step 3: Run the full test suite** + +Run: `source .venv/bin/activate && pytest tests/test_main.py -v` +Expected: all tests PASS + +- [ ] **Step 4: Commit** + +```bash +git add tests/test_main.py +git commit -m "Test e2e : recherche → observatoire #65" +``` From 1d88682f85394fb478c8708f9f708317e8277318 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:43:10 +0100 Subject: [PATCH 28/52] Ajout du lien observatoire dans acheteur_nom via add_links() #65 Co-Authored-By: Claude Sonnet 4.6 --- src/utils.py | 24 +++++++++++++++--------- tests/test_main.py | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/utils.py b/src/utils.py index 642bad4..09eca00 100644 --- a/src/utils.py +++ b/src/utils.py @@ -80,15 +80,21 @@ def add_links(dff: pl.DataFrame): .alias(col) ) if col.startswith("acheteur_"): - dff = dff.with_columns( - ( - '' - + pl.col(col) - + "" - ).alias(col) + detail_link = ( + '' + + pl.col(col) + + "" ) + if col == "acheteur_nom": + detail_link = ( + detail_link + + ' 📊' + ) + dff = dff.with_columns(detail_link.alias(col)) if col == "uid": dff = dff.with_columns( ( @@ -648,7 +654,7 @@ def prepare_table_data( # Remplace les strings null par "", mais pas les numeric null dff = dff.fill_null("") - # Ajout des liens vers l'annuaire des entreprises + # Ajout des liens vers les pages de détails dff = add_links(dff) # Ajout des liens vers les fichiers Open Data diff --git a/tests/test_main.py b/tests/test_main.py index 1b8e7db..39b38a2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -29,12 +29,11 @@ def test_001_logo_and_search(dash_duo: DashComposite): assert len(result_table.find_elements(by=By.TAG_NAME, value="tr")) == 2, ( "The search should return only one result" ) # header row + 1 result - assert ( - result_table.find_element( - by=By.CSS_SELECTOR, value=f'td[data-dash-column="{org_type}_nom"]' - ).text - == name - ), f"The search result should have the right {org_type} name" + assert result_table.find_element( + by=By.CSS_SELECTOR, value=f'td[data-dash-column="{org_type}_nom"]' + ).text.startswith(name), ( + f"The search result should have the right {org_type} name" + ) def test_002_filter_persistence(dash_duo: DashComposite): @@ -87,3 +86,28 @@ def test_003_tableau_download(dash_duo: DashComposite): ) assert output["type"] is None assert output["base64"] is True + + +def test_004_add_links_observatoire_acheteur(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "acheteur_id": ["a1"], + "acheteur_nom": ["ACHETEUR 1"], + } + ) + result = add_links(dff) + nom_value = result["acheteur_nom"][0] + id_value = result["acheteur_id"][0] + + # acheteur_nom should contain detail link + observatoire link + assert "/acheteurs/a1" in nom_value + assert "ACHETEUR 1" in nom_value + assert "/observatoire?acheteur_id=a1" in nom_value + assert "📊" in nom_value + + # acheteur_id should NOT contain observatoire link + assert "/observatoire" not in id_value From bf1791635f45d809980e84ae0e5b21299dc72ccd Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:48:05 +0100 Subject: [PATCH 29/52] Ajout du lien observatoire dans titulaire_nom via add_links() #65 Co-Authored-By: Claude Sonnet 4.6 --- src/utils.py | 22 +++++++++++++++------- tests/test_main.py | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/utils.py b/src/utils.py index 09eca00..d791cf9 100644 --- a/src/utils.py +++ b/src/utils.py @@ -62,6 +62,20 @@ def add_links(dff: pl.DataFrame): for col in ["uid", "acheteur_nom", "titulaire_nom", "acheteur_id", "titulaire_id"]: if col in dff.columns: if col.startswith("titulaire_"): + detail_link = ( + '' + + pl.col(col) + + "" + ) + if col == "titulaire_nom": + detail_link = ( + detail_link + + ' 📊' + ) dff = dff.with_columns( pl.when( pl.Expr.or_( @@ -69,13 +83,7 @@ def add_links(dff: pl.DataFrame): pl.col("titulaire_typeIdentifiant") == "SIRET", ) ) - .then( - '' - + pl.col(col) - + "" - ) + .then(detail_link) .otherwise(pl.col(col)) .alias(col) ) diff --git a/tests/test_main.py b/tests/test_main.py index 39b38a2..e3f77bb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -111,3 +111,29 @@ def test_004_add_links_observatoire_acheteur(): # acheteur_id should NOT contain observatoire link assert "/observatoire" not in id_value + + +def test_005_add_links_observatoire_titulaire(): + import polars as pl + + from src.utils import add_links + + dff = pl.DataFrame( + { + "titulaire_id": ["t1"], + "titulaire_nom": ["TITULAIRE 1"], + "titulaire_typeIdentifiant": ["SIRET"], + } + ) + result = add_links(dff) + nom_value = result["titulaire_nom"][0] + id_value = result["titulaire_id"][0] + + # titulaire_nom should contain detail link + observatoire link + assert "/titulaires/t1" in nom_value + assert "TITULAIRE 1" in nom_value + assert "/observatoire?titulaire_id=t1" in nom_value + assert "📊" in nom_value + + # titulaire_id should NOT contain observatoire link + assert "/observatoire" not in id_value From 3745f6df74c50ef5371849ccac6b2913b4c5a3b4 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:55:41 +0100 Subject: [PATCH 30/52] =?UTF-8?q?Callback=20URL=20=E2=86=92=20filtres=20su?= =?UTF-8?q?r=20la=20page=20observatoire=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 34 ++++++++++++++++++++++++++++++++-- tests/conftest.py | 5 +++++ tests/test_main.py | 20 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index d48518c..67ab659 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -1,9 +1,21 @@ +import urllib.parse from datetime import datetime, timedelta import dash_bootstrap_components as dbc import polars as pl import polars.selectors as cs -from dash import ALL, Input, Output, State, callback, ctx, dcc, html, register_page +from dash import ( + ALL, + Input, + Output, + State, + callback, + ctx, + dcc, + html, + no_update, + register_page, +) from src.figures import ( get_barchart_sources, @@ -107,7 +119,7 @@ def _apply_filters( layout = [ dcc.Store(id="dashboard-filters"), - dcc.Location(id="dashboard_url"), + dcc.Location(id="dashboard_url", refresh="callback-nav"), dbc.Modal( [ dbc.ModalHeader(dbc.ModalTitle("Montants")), @@ -280,6 +292,24 @@ Alors, on fait comment ? ] +@callback( + Output("dashboard_acheteur_id", "value"), + Output("dashboard_titulaire_id", "value"), + Output("dashboard_url", "search"), + Input("dashboard_url", "search"), +) +def restore_filters_from_url(search): + if not search: + return no_update, no_update, no_update + + params = urllib.parse.parse_qs(search.lstrip("?")) + + acheteur_id = params.get("acheteur_id", [None])[0] or no_update + titulaire_id = params.get("titulaire_id", [None])[0] or no_update + + return acheteur_id, titulaire_id, "" + + @callback( Output("cards", "children"), Output("btn-download-observatoire", "disabled"), diff --git a/tests/conftest.py b/tests/conftest.py index b14e0f8..f0f2b00 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,11 @@ def test_data(): "sourceFile": "test.xml", "sourceDataset": "test_dataset", "datePublicationDonnees": datetime.date(2025, 1, 1), + "considerationsSociales": "", + "considerationsEnvironnementales": "", + "type": "Marché", + "acheteur_categorie": "Collectivité", + "titulaire_categorie": "PME", } ] path = "tests/test.parquet" diff --git a/tests/test_main.py b/tests/test_main.py index e3f77bb..995a669 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -137,3 +137,23 @@ def test_005_add_links_observatoire_titulaire(): # titulaire_id should NOT contain observatoire link assert "/observatoire" not in id_value + + +def test_006_observatoire_url_to_input(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + import time + + time.sleep(1) # Allow callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated from URL param" + ) From e804b6bca2291d1ac3dff9208cd8fd1dc9ba6802 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 14:59:32 +0100 Subject: [PATCH 31/52] URL partageable pour la page observatoire #65 Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 52 +++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 22 +++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 67ab659..b42a99a 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -274,6 +274,12 @@ Alors, on fait comment ? disabled=True, className="mt-2", ), + dcc.Input( + id="observatoire-share-url", + readOnly=True, + style={"display": "none"}, + ), + html.Div(id="observatoire-copy-container"), ], ), dbc.Col( @@ -310,6 +316,52 @@ def restore_filters_from_url(search): return acheteur_id, titulaire_id, "" +@callback( + Output("observatoire-share-url", "value"), + Output("observatoire-copy-container", "children"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_titulaire_id", "value"), + State("dashboard_url", "href"), + prevent_initial_call=True, +) +def sync_observatoire_share_url(acheteur_id, titulaire_id, href): + if not href: + return no_update, no_update + + base_url = href.split("?")[0] + + params = {} + if acheteur_id: + params["acheteur_id"] = acheteur_id + if titulaire_id: + params["titulaire_id"] = titulaire_id + + query_string = urllib.parse.urlencode(params) + full_url = f"{base_url}?{query_string}" if query_string else base_url + + copy_button = dcc.Clipboard( + id="btn-copy-observatoire-url", + target_id="observatoire-share-url", + title="Copier l'URL de cette vue", + style={ + "display": "inline-block", + "fontSize": 20, + "verticalAlign": "top", + "cursor": "pointer", + }, + className="fa fa-link", + children=[ + dbc.Button( + "Partager", + className="btn btn-primary mt-2", + title="Copier l'adresse de cette vue filtrée pour la partager.", + ) + ], + ) + + return full_url, copy_button + + @callback( Output("cards", "children"), Output("btn-download-observatoire", "disabled"), diff --git a/tests/test_main.py b/tests/test_main.py index 995a669..6cd2903 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -157,3 +157,25 @@ def test_006_observatoire_url_to_input(dash_duo: DashComposite): assert acheteur_input.get_attribute("value") == "a1", ( "acheteur_id input should be populated from URL param" ) + + +def test_007_observatoire_share_url(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Navigate to observatoire with acheteur_id query param + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_element("#observatoire-share-url", timeout=4) + + import time + + time.sleep(1) # Allow callback chain to complete + + share_url_input = dash_duo.find_element("#observatoire-share-url") + share_url_value = share_url_input.get_attribute("value") + + assert "acheteur_id=a1" in share_url_value, ( + f"Share URL should contain acheteur_id param, got: {share_url_value}" + ) From acb8500dc0dece7a13536e093e79848201dc72fd Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 15:02:27 +0100 Subject: [PATCH 32/52] =?UTF-8?q?Test=20e2e=20:=20recherche=20=E2=86=92=20?= =?UTF-8?q?observatoire=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- tests/test_main.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 6cd2903..6809636 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -179,3 +179,38 @@ def test_007_observatoire_share_url(dash_duo: DashComposite): assert "acheteur_id=a1" in share_url_value, ( f"Share URL should contain acheteur_id param, got: {share_url_value}" ) + + +def test_008_search_to_observatoire(dash_duo: DashComposite): + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Search for an acheteur + search_bar = dash_duo.find_element("#search") + search_bar.send_keys("ACHETEUR 1") + search_bar.send_keys(Keys.ENTER) + + dash_duo.wait_for_element("#results_acheteur_datatable", timeout=2) + + # Find the observatoire link in acheteur_nom column + observatoire_link = dash_duo.find_element( + '#results_acheteur_datatable td[data-dash-column="acheteur_nom"] a[href*="observatoire"]' + ) + assert "📊" in observatoire_link.text + + # Click the observatoire link + observatoire_link.click() + + # Wait for observatoire page to load + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + import time + + time.sleep(1) # Allow callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "a1", ( + "acheteur_id input should be populated after navigating from search" + ) From 958c3956ea8ee57750e747d2d4cd8f1834819081 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 16:05:29 +0100 Subject: [PATCH 33/52] =?UTF-8?q?Correction=20des=20probl=C3=A8mes=20de=20?= =?UTF-8?q?double=20reload=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index b42a99a..6887f93 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -301,19 +301,25 @@ Alors, on fait comment ? @callback( Output("dashboard_acheteur_id", "value"), Output("dashboard_titulaire_id", "value"), - Output("dashboard_url", "search"), + # Output("dashboard_url", "search"), Input("dashboard_url", "search"), ) def restore_filters_from_url(search): if not search: - return no_update, no_update, no_update + return no_update, no_update params = urllib.parse.parse_qs(search.lstrip("?")) - acheteur_id = params.get("acheteur_id", [None])[0] or no_update - titulaire_id = params.get("titulaire_id", [None])[0] or no_update + def get_param_value(key): + values = params.get(key) + if values and values[0] is not None: + return values[0] + return no_update - return acheteur_id, titulaire_id, "" + acheteur_id = get_param_value("acheteur_id") + titulaire_id = get_param_value("titulaire_id") + + return acheteur_id, titulaire_id @callback( From 5d4c0b8438e1e6f507995a2bd047ba68124dc8c5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:31:45 +0100 Subject: [PATCH 34/52] test: failing test for observatoire localStorage filter persistence #65 --- tests/test_main.py | 75 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 6809636..f9d36b7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -51,7 +51,7 @@ def test_002_filter_persistence(dash_duo: DashComposite): _filter_input: WebElement = dash_duo.find_element(filter_input_selector) return _filter_input - for page in ["tableau", "acheteurs/a1", "titulaires/t1"]: + for page in ["tableau", "acheteurs/123", "titulaires/345"]: print("page:", page) filter_input = open_page_and_check_filter_input() filter_input.send_keys("11") # a UID that doesn't exist @@ -73,8 +73,8 @@ def test_003_tableau_download(dash_duo: DashComposite): outputs = [ download_data(1, "", [], None), - download_acheteur_data(1, dicts, "a1", "2025"), - download_titulaire_data(1, dicts, "t1", "2025"), + download_acheteur_data(1, dicts, "123", "2025"), + download_titulaire_data(1, dicts, "345", "2025"), ] for output in outputs: assert isinstance(output, dict) @@ -95,7 +95,7 @@ def test_004_add_links_observatoire_acheteur(): dff = pl.DataFrame( { - "acheteur_id": ["a1"], + "acheteur_id": ["123"], "acheteur_nom": ["ACHETEUR 1"], } ) @@ -104,9 +104,9 @@ def test_004_add_links_observatoire_acheteur(): id_value = result["acheteur_id"][0] # acheteur_nom should contain detail link + observatoire link - assert "/acheteurs/a1" in nom_value + assert "/acheteurs/123" in nom_value assert "ACHETEUR 1" in nom_value - assert "/observatoire?acheteur_id=a1" in nom_value + assert "/observatoire?acheteur_id=123" in nom_value assert "📊" in nom_value # acheteur_id should NOT contain observatoire link @@ -120,7 +120,7 @@ def test_005_add_links_observatoire_titulaire(): dff = pl.DataFrame( { - "titulaire_id": ["t1"], + "titulaire_id": ["345"], "titulaire_nom": ["TITULAIRE 1"], "titulaire_typeIdentifiant": ["SIRET"], } @@ -130,9 +130,9 @@ def test_005_add_links_observatoire_titulaire(): id_value = result["titulaire_id"][0] # titulaire_nom should contain detail link + observatoire link - assert "/titulaires/t1" in nom_value + assert "/titulaires/345" in nom_value assert "TITULAIRE 1" in nom_value - assert "/observatoire?titulaire_id=t1" in nom_value + assert "/observatoire?titulaire_id=345" in nom_value assert "📊" in nom_value # titulaire_id should NOT contain observatoire link @@ -146,7 +146,7 @@ def test_006_observatoire_url_to_input(dash_duo: DashComposite): dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) # Navigate to observatoire with acheteur_id query param - dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=123") dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) import time @@ -154,7 +154,7 @@ def test_006_observatoire_url_to_input(dash_duo: DashComposite): time.sleep(1) # Allow callback chain to complete acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") - assert acheteur_input.get_attribute("value") == "a1", ( + assert acheteur_input.get_attribute("value") == "123", ( "acheteur_id input should be populated from URL param" ) @@ -166,7 +166,7 @@ def test_007_observatoire_share_url(dash_duo: DashComposite): dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) # Navigate to observatoire with acheteur_id query param - dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=a1") + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=123") dash_duo.wait_for_element("#observatoire-share-url", timeout=4) import time @@ -176,7 +176,7 @@ def test_007_observatoire_share_url(dash_duo: DashComposite): share_url_input = dash_duo.find_element("#observatoire-share-url") share_url_value = share_url_input.get_attribute("value") - assert "acheteur_id=a1" in share_url_value, ( + assert "acheteur_id=123" in share_url_value, ( f"Share URL should contain acheteur_id param, got: {share_url_value}" ) @@ -211,6 +211,53 @@ def test_008_search_to_observatoire(dash_duo: DashComposite): time.sleep(1) # Allow callback chain to complete acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") - assert acheteur_input.get_attribute("value") == "a1", ( + assert acheteur_input.get_attribute("value") == "123", ( "acheteur_id input should be populated after navigating from search" ) + + +def test_009_observatoire_filter_persistence(dash_duo: DashComposite): + import time + + from src.app import app + + dash_duo.start_server(app) + dash_duo.wait_for_text_to_equal(".logo > h1", "decp.info", timeout=4) + + # Clear localStorage to start from a clean state + dash_duo.driver.execute_script("localStorage.clear()") + + # Navigate to observatoire without URL params + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + + # Set the acheteur_id text input; press Enter to trigger the debounced save callback + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + dash_duo.clear_input(acheteur_input) + acheteur_input.send_keys("123") + acheteur_input.send_keys(Keys.ENTER) + + time.sleep(0.3) # allow the save callback to write to localStorage + + # Navigate away + dash_duo.wait_for_page(f"{dash_duo.server_url}/") + + # Navigate back without URL params + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + time.sleep(0.5) # allow restore callback chain to complete + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "123", ( + "acheteur_id should be restored from localStorage after navigating back" + ) + + # Also verify URL params still override localStorage + dash_duo.wait_for_page(f"{dash_duo.server_url}/observatoire?acheteur_id=123") + dash_duo.wait_for_element("#dashboard_acheteur_id", timeout=4) + time.sleep(0.5) + + acheteur_input = dash_duo.find_element("#dashboard_acheteur_id") + assert acheteur_input.get_attribute("value") == "123", ( + "URL param acheteur_id should override the value stored in localStorage" + ) From 1fdcb12dd25918008b520df45a5bb172833f2805 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:32:43 +0100 Subject: [PATCH 35/52] feat: add dcc.Store and debounce text inputs on observatoire page #65 --- src/pages/observatoire.py | 50 +++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 6887f93..ec48c38 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -27,6 +27,8 @@ from src.figures import ( from src.utils import ( departements, df, + df_acheteurs, + df_titulaires, format_number, get_enum_values_as_dict, meta_content, @@ -118,8 +120,8 @@ def _apply_filters( layout = [ - dcc.Store(id="dashboard-filters"), dcc.Location(id="dashboard_url", refresh="callback-nav"), + dcc.Store(id="observatoire-filters", storage_type="local"), dbc.Modal( [ dbc.ModalHeader(dbc.ModalTitle("Montants")), @@ -146,7 +148,7 @@ Alors, on fait comment ? html.Div( className="container-fluid", children=[ - html.H2(name), + html.H2(children=[name], id="page_title"), dcc.Loading( overlay_style={"visibility": "visible", "filter": "blur(2px)"}, id="loading-statistques", @@ -175,6 +177,7 @@ Alors, on fait comment ? dcc.Input( id="dashboard_acheteur_id", placeholder="SIRET", + debounce=True, style={"width": "100%"}, ), ), @@ -207,6 +210,7 @@ Alors, on fait comment ? dcc.Input( id="dashboard_titulaire_id", placeholder="SIRET", + debounce=True, style={"width": "100%"}, ), ), @@ -473,7 +477,11 @@ def udpate_dashboard_cards( ) cards.append( make_card( - title="Catégorie d'acheteur", fig=donut_acheteur_categorie, lg=12, xl=8 + title="Catégorie d'acheteur", + subtitle="en nombre de marchés attribués", + fig=donut_acheteur_categorie, + lg=12, + xl=8, ) ) @@ -481,11 +489,21 @@ def udpate_dashboard_cards( lff, "titulaire_categorie", per_uid=False, nulls="?" ) cards.append( - make_card(title="Catégorie d'entreprise", fig=donut_titulaire_categorie) + make_card( + title="Catégorie d'entreprise", + subtitle="en nombre de marchés attribués", + fig=donut_titulaire_categorie, + ) ) donut_marche_type = make_donut(lff, "type", per_uid=True, nulls="?") - cards.append(make_card(title="Type d'achat", fig=donut_marche_type)) + cards.append( + make_card( + title="Type d'achat", + subtitle="en nombre de marchés attribués", + fig=donut_marche_type, + ) + ) geographic_maps: list[dbc.Col] = get_geographic_maps(dff) @@ -573,3 +591,25 @@ def download_observatoire( ) def toggle_montant_modal(n_triggers, _close): return isinstance(ctx.triggered_id, dict) and any(n_triggers) + + +@callback( + Output("page_title", "children"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_titulaire_id", "value"), + prevent_initial_call=False, +) +def add_organization_name_in_title(acheteur_id, titulaire_id): + def lookup_nom(df_org, id_col, nom_col, org_id): + match = df_org.filter(pl.col(id_col) == org_id) + return match[nom_col].item(0) if match.height >= 1 else None + + if acheteur_id and len(acheteur_id) == 12: + if nom := lookup_nom(df_acheteurs, "acheteur_id", "acheteur_nom", acheteur_id): + return f"{name} - {nom}" + elif titulaire_id and len(titulaire_id) == 12: + if nom := lookup_nom( + df_titulaires, "titulaire_id", "titulaire_nom", titulaire_id + ): + return f"{name} - {nom}" + return name From 051e908bd18f983d3f3a70951dc5d4f17c7b3b8c Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:33:15 +0100 Subject: [PATCH 36/52] feat: save observatoire filters to localStorage on change #65 --- src/pages/observatoire.py | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index ec48c38..53dfddf 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -326,6 +326,46 @@ def restore_filters_from_url(search): return acheteur_id, titulaire_id +@callback( + Output("observatoire-filters", "data"), + Input("dashboard_year", "value"), + Input("dashboard_acheteur_id", "value"), + Input("dashboard_acheteur_categorie", "value"), + Input("dashboard_acheteur_departement_code", "value"), + Input("dashboard_titulaire_id", "value"), + Input("dashboard_titulaire_categorie", "value"), + Input("dashboard_titulaire_departement_code", "value"), + Input("dashboard_marche_type", "value"), + Input("dashboard_marche_considerationsSociales", "value"), + Input("dashboard_marche_considerationsEnvironnementales", "value"), + prevent_initial_call=True, +) +def save_filters_to_storage( + year, + acheteur_id, + acheteur_categorie, + acheteur_departement_code, + titulaire_id, + titulaire_categorie, + titulaire_departement_code, + marche_type, + considerations_sociales, + considerations_environnementales, +): + return { + "year": year, + "acheteur_id": acheteur_id, + "acheteur_categorie": acheteur_categorie, + "acheteur_departement_code": acheteur_departement_code, + "titulaire_id": titulaire_id, + "titulaire_categorie": titulaire_categorie, + "titulaire_departement_code": titulaire_departement_code, + "marche_type": marche_type, + "considerations_sociales": considerations_sociales, + "considerations_environnementales": considerations_environnementales, + } + + @callback( Output("observatoire-share-url", "value"), Output("observatoire-copy-container", "children"), From b4a42449ad5a832fa6c2ea172b30f419255db3aa Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:35:31 +0100 Subject: [PATCH 37/52] feat: restore observatoire filters from localStorage on page load #65 --- src/pages/observatoire.py | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 53dfddf..aa13f18 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -303,27 +303,54 @@ Alors, on fait comment ? @callback( + Output("dashboard_year", "value"), Output("dashboard_acheteur_id", "value"), + Output("dashboard_acheteur_categorie", "value"), + Output("dashboard_acheteur_departement_code", "value"), Output("dashboard_titulaire_id", "value"), - # Output("dashboard_url", "search"), + Output("dashboard_titulaire_categorie", "value"), + Output("dashboard_titulaire_departement_code", "value"), + Output("dashboard_marche_type", "value"), + Output("dashboard_marche_considerationsSociales", "value"), + Output("dashboard_marche_considerationsEnvironnementales", "value"), Input("dashboard_url", "search"), + Input("dashboard_url", "pathname"), + State("observatoire-filters", "data"), ) -def restore_filters_from_url(search): - if not search: - return no_update, no_update +def restore_filters(search, _pathname, stored_filters): + if search: + params = urllib.parse.parse_qs(search.lstrip("?")) + acheteur_id = (params.get("acheteur_id") or [None])[0] or None + titulaire_id = (params.get("titulaire_id") or [None])[0] or None + if acheteur_id or titulaire_id: + return ( + None, + acheteur_id, + None, + None, + titulaire_id, + None, + None, + None, + None, + None, + ) - params = urllib.parse.parse_qs(search.lstrip("?")) + if stored_filters: + return ( + stored_filters.get("year"), + stored_filters.get("acheteur_id"), + stored_filters.get("acheteur_categorie"), + stored_filters.get("acheteur_departement_code"), + stored_filters.get("titulaire_id"), + stored_filters.get("titulaire_categorie"), + stored_filters.get("titulaire_departement_code"), + stored_filters.get("marche_type"), + stored_filters.get("considerations_sociales"), + stored_filters.get("considerations_environnementales"), + ) - def get_param_value(key): - values = params.get(key) - if values and values[0] is not None: - return values[0] - return no_update - - acheteur_id = get_param_value("acheteur_id") - titulaire_id = get_param_value("titulaire_id") - - return acheteur_id, titulaire_id + return (no_update,) * 10 @callback( From f8fffb6fa444afeef104314036c95e6c78841362 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:40:59 +0100 Subject: [PATCH 38/52] get_top_org un peu plus configurable #65 --- src/callbacks.py | 8 ++++---- src/pages/acheteur.py | 2 +- src/pages/titulaire.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/callbacks.py b/src/callbacks.py index 0ca434e..9d791db 100644 --- a/src/callbacks.py +++ b/src/callbacks.py @@ -5,14 +5,14 @@ from src.figures import DataTable from utils import add_links_in_dict, format_values, setup_table_columns -def get_top_org_table(data, org_type: str): +def get_top_org_table(data, org_type: str, extra_columns: list): dff = pl.DataFrame(data, strict=False, infer_schema_length=5000) if dff.height == 0: return html.Div() - dff = dff.select( - ["uid", f"{org_type}_id", f"{org_type}_nom", "titulaire_distance", "montant"] - ) + extra_columns = [] if extra_columns is None else extra_columns + + dff = dff.select(["uid", f"{org_type}_id", f"{org_type}_nom"] + extra_columns) dff_nb = dff.group_by( f"{org_type}_id", f"{org_type}_nom", "titulaire_distance" ).agg(pl.len().alias("Attributions"), pl.sum("montant").alias("montant")) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 199c757..76cd2d6 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -340,7 +340,7 @@ def get_last_marches_data( Input(component_id="acheteur_data", component_property="data"), ) def get_top_titulaires(data): - return get_top_org_table(data, "titulaire") + return get_top_org_table(data, "titulaire", ["titulaire_distance", "montant"]) @callback( diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index b7e9001..ecee685 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -354,7 +354,7 @@ def get_last_marches_data( Input(component_id="titulaire_data", component_property="data"), ) def get_top_acheteurs(data): - return get_top_org_table(data, "acheteur") + return get_top_org_table(data, "acheteur", ["titulaire_distance", "montant"]) @callback( From 26dd2aaf1a1218a90d749809f64438289dfe7e5e Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:41:46 +0100 Subject: [PATCH 39/52] Correction de l'injection du nom d'org dans le titre #65 --- src/figures.py | 5 ----- src/pages/observatoire.py | 4 ++-- tests/conftest.py | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/figures.py b/src/figures.py index 8337f9c..8e8afbf 100644 --- a/src/figures.py +++ b/src/figures.py @@ -89,11 +89,6 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str): .sort(by=[type_date, "len"], descending=True) ) - # lff = lff.with_columns( - # pl.when(pl.col("sourceDataset").is_null()).then( - # pl.lit("Source inconnue")).alias("sourceDataset") - # ) - lff = lff.sort(by=["sourceDataset"], descending=False) dff: pl.DataFrame = lff.collect(engine="streaming") diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index aa13f18..404e14c 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -671,10 +671,10 @@ def add_organization_name_in_title(acheteur_id, titulaire_id): match = df_org.filter(pl.col(id_col) == org_id) return match[nom_col].item(0) if match.height >= 1 else None - if acheteur_id and len(acheteur_id) == 12: + if acheteur_id and len(acheteur_id) == 14: if nom := lookup_nom(df_acheteurs, "acheteur_id", "acheteur_nom", acheteur_id): return f"{name} - {nom}" - elif titulaire_id and len(titulaire_id) == 12: + elif titulaire_id and len(titulaire_id) == 14: if nom := lookup_nom( df_titulaires, "titulaire_id", "titulaire_nom", titulaire_id ): diff --git a/tests/conftest.py b/tests/conftest.py index f0f2b00..78f1362 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,9 +13,9 @@ def test_data(): "uid": "1", "id": "1", "acheteur_nom": "ACHETEUR 1", - "acheteur_id": "a1", + "acheteur_id": "123", "titulaire_nom": "TITULAIRE 1", - "titulaire_id": "t1", + "titulaire_id": "345", "montant": 10, "dateNotification": datetime.date(2025, 1, 1), "codeCPV": "71600000", From f2046d6ba70f60831e2bee9291badeb855e6a784 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:46:38 +0100 Subject: [PATCH 40/52] =?UTF-8?q?Meilleure=20int=C3=A9gration=20du=20nom?= =?UTF-8?q?=20de=20l'org=20#65?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/observatoire.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 404e14c..87c3470 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -673,10 +673,16 @@ def add_organization_name_in_title(acheteur_id, titulaire_id): if acheteur_id and len(acheteur_id) == 14: if nom := lookup_nom(df_acheteurs, "acheteur_id", "acheteur_nom", acheteur_id): - return f"{name} - {nom}" + return [ + name, + html.Small(nom, className="text-muted d-block fw-normal fs-5"), + ] elif titulaire_id and len(titulaire_id) == 14: if nom := lookup_nom( df_titulaires, "titulaire_id", "titulaire_nom", titulaire_id ): - return f"{name} - {nom}" + return [ + name, + html.Small(nom, className="text-muted d-block fw-normal fs-5"), + ] return name From c77511d4e85fe63c48b2bdc00097828265c07f93 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 20:57:30 +0100 Subject: [PATCH 41/52] Taille de donut flexible #65 --- src/figures.py | 4 ++++ src/pages/observatoire.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/figures.py b/src/figures.py index 8e8afbf..451f24c 100644 --- a/src/figures.py +++ b/src/figures.py @@ -609,6 +609,7 @@ def make_donut( names_col, per_uid: bool, nulls="?", + potentially_many_names: bool = False, ): title = data_schema[names_col]["title"] lff = lff.rename({names_col: title}) @@ -620,6 +621,7 @@ def make_donut( lff = lff.group_by(title).len("Nombre") lff = lff.with_columns(pl.col(title).replace(None, pl.lit(nulls))) dff = lff.collect(engine="streaming") + nb_names = dff[title].n_unique() dff = dff.with_columns( pl.col("Nombre") .map_elements(format_number, return_dtype=pl.String) @@ -639,6 +641,8 @@ def make_donut( ) fig = fig.update_layout(showlegend=False, font=dict(size=14)) graph = dcc.Graph(figure=fig) + if potentially_many_names: + return graph, nb_names return graph diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 87c3470..6ca7d10 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -539,16 +539,20 @@ def udpate_dashboard_cards( cards.append(make_card(title="Résumé", paragraphs=card_basic_counts)) - donut_acheteur_categorie = make_donut( - lff, "acheteur_categorie", nulls="Autres", per_uid=True + donut_acheteur_categorie, nb_acheteur_categories = make_donut( + lff, + "acheteur_categorie", + nulls="Autres", + per_uid=True, + potentially_many_names=True, ) cards.append( make_card( title="Catégorie d'acheteur", subtitle="en nombre de marchés attribués", fig=donut_acheteur_categorie, - lg=12, - xl=8, + lg=12 if nb_acheteur_categories > 4 else 6, + xl=8 if nb_acheteur_categories > 4 else 4, ) ) From 78d528f75bf7b3f7f46a08f12d36d513960d54e6 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 21:13:21 +0100 Subject: [PATCH 42/52] Filtre par montant #65 --- src/assets/css/style.css | 13 ++++++++- src/pages/observatoire.py | 60 ++++++++++++++++++++++++++++++++++++++- tests/test_main.py | 41 ++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 12dd156..f93c94a 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -201,7 +201,8 @@ p.version > a { margin-bottom: 6px; } -#filters input[type="text"] { +#filters input[type="text"], +#filters input[type="number"] { border: 1px #ccc solid; border-radius: 3px; padding-left: 8px; @@ -578,3 +579,13 @@ summary > h4 { display: none; } } + +input[type="number"]::-webkit-outer-spin-button, +input[type="number"]::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; +} + +input[type="number"] { + -moz-appearance: textfield; +} diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 6ca7d10..f2bbe65 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -67,6 +67,8 @@ def _apply_filters( marche_type, considerations_sociales, considerations_environnementales, + montant_min=None, + montant_max=None, ) -> pl.LazyFrame: if year: lff = lff.filter(pl.col("dateNotification").dt.year() == int(year)) @@ -116,6 +118,12 @@ def _apply_filters( > 0 ) + if montant_min is not None: + lff = lff.filter(pl.col("montant") >= montant_min) + + if montant_max is not None: + lff = lff.filter(pl.col("montant") <= montant_max) + return lff @@ -271,6 +279,32 @@ Alors, on fait comment ? ), ), ), + dbc.Row( + [ + dbc.Col( + dcc.Input( + id="dashboard_montant_min", + placeholder="Montant min.", + type="number", + min=0, + debounce=True, + style={"width": "100%"}, + ), + width=6, + ), + dbc.Col( + dcc.Input( + id="dashboard_montant_max", + placeholder="Montant max.", + type="number", + min=0, + debounce=True, + style={"width": "100%"}, + ), + width=6, + ), + ] + ), dcc.Download(id="download-observatoire"), dbc.Button( "Télécharger au format Excel", @@ -313,6 +347,8 @@ Alors, on fait comment ? Output("dashboard_marche_type", "value"), Output("dashboard_marche_considerationsSociales", "value"), Output("dashboard_marche_considerationsEnvironnementales", "value"), + Output("dashboard_montant_min", "value"), + Output("dashboard_montant_max", "value"), Input("dashboard_url", "search"), Input("dashboard_url", "pathname"), State("observatoire-filters", "data"), @@ -334,6 +370,8 @@ def restore_filters(search, _pathname, stored_filters): None, None, None, + None, + None, ) if stored_filters: @@ -348,9 +386,11 @@ def restore_filters(search, _pathname, stored_filters): stored_filters.get("marche_type"), stored_filters.get("considerations_sociales"), stored_filters.get("considerations_environnementales"), + stored_filters.get("montant_min"), + stored_filters.get("montant_max"), ) - return (no_update,) * 10 + return (no_update,) * 12 @callback( @@ -365,6 +405,8 @@ def restore_filters(search, _pathname, stored_filters): Input("dashboard_marche_type", "value"), Input("dashboard_marche_considerationsSociales", "value"), Input("dashboard_marche_considerationsEnvironnementales", "value"), + Input("dashboard_montant_min", "value"), + Input("dashboard_montant_max", "value"), prevent_initial_call=True, ) def save_filters_to_storage( @@ -378,6 +420,8 @@ def save_filters_to_storage( marche_type, considerations_sociales, considerations_environnementales, + montant_min, + montant_max, ): return { "year": year, @@ -390,6 +434,8 @@ def save_filters_to_storage( "marche_type": marche_type, "considerations_sociales": considerations_sociales, "considerations_environnementales": considerations_environnementales, + "montant_min": montant_min, + "montant_max": montant_max, } @@ -453,6 +499,8 @@ def sync_observatoire_share_url(acheteur_id, titulaire_id, href): Input("dashboard_marche_type", "value"), Input("dashboard_marche_considerationsSociales", "value"), Input("dashboard_marche_considerationsEnvironnementales", "value"), + Input("dashboard_montant_min", "value"), + Input("dashboard_montant_max", "value"), ) def udpate_dashboard_cards( dashboard_year, @@ -465,6 +513,8 @@ def udpate_dashboard_cards( dashboard_marche_type, dashboard_marche_considerations_sociales, dashboard_marche_considerations_environnementales, + dashboard_montant_min, + dashboard_montant_max, ): lff: pl.LazyFrame = df.lazy() lff = lff.select( @@ -490,6 +540,8 @@ def udpate_dashboard_cards( dashboard_marche_type, dashboard_marche_considerations_sociales, dashboard_marche_considerations_environnementales, + montant_min=dashboard_montant_min, + montant_max=dashboard_montant_max, ) # Génération des métriques @@ -618,6 +670,8 @@ def udpate_dashboard_cards( State("dashboard_marche_type", "value"), State("dashboard_marche_considerationsSociales", "value"), State("dashboard_marche_considerationsEnvironnementales", "value"), + State("dashboard_montant_min", "value"), + State("dashboard_montant_max", "value"), prevent_initial_call=True, ) def download_observatoire( @@ -632,6 +686,8 @@ def download_observatoire( marche_type, considerations_sociales, considerations_environnementales, + montant_min, + montant_max, ): lff = _apply_filters( df.lazy(), @@ -645,6 +701,8 @@ def download_observatoire( marche_type, considerations_sociales, considerations_environnementales, + montant_min=montant_min, + montant_max=montant_max, ) def to_bytes(buffer): diff --git a/tests/test_main.py b/tests/test_main.py index f9d36b7..6a13762 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -216,6 +216,47 @@ def test_008_search_to_observatoire(dash_duo: DashComposite): ) +def test_010_observatoire_montant_filter(): + import datetime + + import polars as pl + + from pages.observatoire import _apply_filters + from src.app import ( + app, # noqa: F401 – instantiates the Dash app before register_page() calls + ) + + data = pl.DataFrame( + { + "uid": ["1", "2", "3"], + "montant": [100.0, 500.0, 1000.0], + "dateNotification": [datetime.date(2025, 1, 1)] * 3, + } + ) + + def apply(min_val=None, max_val=None): + return _apply_filters( + data.lazy(), + year="2025", + acheteur_id=None, + acheteur_categorie=None, + acheteur_departement_code=None, + titulaire_id=None, + titulaire_categorie=None, + titulaire_departement_code=None, + marche_type=None, + considerations_sociales=None, + considerations_environnementales=None, + montant_min=min_val, + montant_max=max_val, + ).collect() + + assert apply().height == 3 + assert apply(min_val=400).height == 2 # 500, 1000 + assert apply(max_val=500).height == 2 # 100, 500 + assert apply(min_val=200, max_val=600).height == 1 # 500 only + + def test_009_observatoire_filter_persistence(dash_duo: DashComposite): import time From 125c520c19da26bf94e047660b75b4d193761f14 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 21:47:29 +0100 Subject: [PATCH 43/52] Claude memory --- CLAUDE.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b4ead7d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,85 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +**decp.info** is a French public procurement data explorer — a Dash (Python) web app for browsing, filtering, and visualizing _Données Essentielles de la Commande Publique_ (DECP). The UI is in French. + +## Commands + +### Setup + +```bash +python -m venv .venv && source .venv/bin/activate +pip install ".[dev]" +cp template.env .env # then customize .env +``` + +### Development + +```bash +uv run run.py # starts Dash with debug=True and hot reload +``` + +### Production + +```bash +gunicorn app:server +``` + +### Tests + +```bash +uv run pytest # run all tests (Selenium-based integration tests) +uv run pytest tests/test_main.py::test_001_logo_and_search # run a single test +``` + +Tests require a running Chrome/Chromium browser. They use `DashComposite` from `dash[testing]` with Selenium WebDriver. + +## Architecture + +### Multi-page Dash app + +- `src/app.py` — creates the Dash app instance, navbar, SEO endpoints (robots.txt, sitemap.xml), Matomo analytics +- `src/pages/*.py` — each page registers itself with `@register_page()` and owns its own layout and callbacks +- `run.py` — dev entry point; exports `server` (Flask) for gunicorn + +### Key pages + +| Page | URL | Purpose | +| ----------------- | --------------- | -------------------------------------- | +| `recherche.py` | `/` | Search homepage for buyers/contractors | +| `acheteur.py` | `/acheteur` | Buyer detail with stats, charts, maps | +| `titulaire.py` | `/titulaire` | Contractor detail | +| `tableau.py` | `/tableau` | Filterable data table with exports | +| `marche.py` | `/marche` | Individual contract detail | +| `observatoire.py` | `/observatoire` | An interactive analytics dashboard | + +### Data layer + +- Data is stored as **Parquet** and loaded with **Polars** (fast columnar operations) +- Path set via `DATA_FILE_PARQUET_PATH` env var; tests use `tests/test.parquet` +- `src/utils.py` — filtering helpers, search (`search_org`), link generation, geographic data loading +- `src/callbacks.py` — shared Dash callbacks (e.g. `get_top_org_table`) +- `src/figures.py` — chart and map components (Plotly Express, Dash Leaflet with marker clustering) +- a Parquet file with production data is located at `../decp-processing/decp_prod.parquet` (~ 1,5 million records) +- the TableSchema of the dataset with the list of field and their definition is located at `../decp-processing/reference/base_schema.json` +- `tests/test.parquet` is very small and may not contain all possible columns, only those necessary for testing + +### UI stack + +- **Dash 3.4** + **Dash Bootstrap Components** for layout +- **Plotly Express** for charts +- **Dash Leaflet** + **Dash Extensions** for interactive maps with clustering +- Custom CSS in `src/assets/css/` + +### Environment + +- `DEVELOPMENT=true` enables debug logging and is set automatically during tests +- `.env` file is required at runtime (copy from `template.env`) + +### Deployment + +- `main` branch → manual deploy to decp.info via GitHub Actions +- `dev` branch → auto-deploy to test.decp.info via GitHub Actions From 9da3d9ce34ed588218252b5dd0d6b4dfa3be03a5 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 21:48:23 +0100 Subject: [PATCH 44/52] =?UTF-8?q?Ne=20pas=20supprimer=20les=20donn=C3=A9es?= =?UTF-8?q?=20de=20test=20=C3=A0=20la=20fin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 78f1362..1874b52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,10 +48,6 @@ def test_data(): pl.DataFrame(data).write_parquet("tests/test.parquet") yield path - if os.path.exists(path): - os.unlink(path) - print(path, "deleted") - def pytest_setup_options(): options = Options() From 1ffa995a4a1c0ffe77fccfdc29c04777a946bd85 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:02:20 +0100 Subject: [PATCH 45/52] docs: spec for distance histogram on observatoire, acheteur, titulaire pages Co-Authored-By: Claude Sonnet 4.6 --- .../2026-03-18-distance-histogram-design.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-18-distance-histogram-design.md diff --git a/docs/superpowers/specs/2026-03-18-distance-histogram-design.md b/docs/superpowers/specs/2026-03-18-distance-histogram-design.md new file mode 100644 index 0000000..091e274 --- /dev/null +++ b/docs/superpowers/specs/2026-03-18-distance-histogram-design.md @@ -0,0 +1,64 @@ +# Distance Histogram — Design Spec + +**Date:** 2026-03-18 +**Branch:** feature/65_observatoire + +## Goal + +Display the distribution of distances (in km) between buyers and winning contractors, to help users assess whether a buyer or contractor tends to deal locally or at a national scale. + +## Data + +- Column: `titulaire_distance` (`Int64`, km) +- Measured at address level — values are always > 0, no zero-handling needed +- Already selected in the observatoire LazyFrame via `cs.starts_with("titulaire")` +- Already available on acheteur and titulaire detail pages + +## Figure Function + +**Location:** `src/figures.py` + +**Signature:** + +```python +def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: +``` + +**Behaviour:** + +- Collects `titulaire_distance` from the LazyFrame, drops nulls +- If the resulting DataFrame is empty after dropping nulls, `px.histogram` produces a blank figure without errors — no guard logic needed. The order of operations must be: drop nulls → log-transform → histogram +- Drop nulls first, then pre-log-transform the column (`pl.col("titulaire_distance").log(10)`) so bins are truly equal-width on a log scale. Use `px.histogram` with `nbins=50` on the transformed values +- Set custom X-axis tick values at powers of 10 (1, 10, 100, 1000, 10000) with km labels, using `fig.update_xaxes(tickvals=[0,1,2,3,4], ticktext=["1","10","100","1 000","10 000"])` +- Y axis: count of contracts +- French axis labels: x = `"Distance (km)"`, y = `"Nombre de marchés"` +- Returns a `dcc.Graph` + +## Integration + +### Observatoire (`src/pages/observatoire.py`) + +- `get_distance_histogram` imported and called inside `udpate_dashboard_cards` +- Result wrapped in `make_card(title="Distance acheteur–titulaire", subtitle="en nombre de marchés, échelle logarithmique", fig=...)` +- Card appended to the `cards` list alongside existing donuts and charts +- No changes to the data pipeline — `titulaire_distance` is already in the LazyFrame + +### Acheteur page (`src/pages/acheteur.py`) + +The acheteur page uses a `dcc.Store` (`acheteur_data`) that holds serialised contract rows as a list of dicts. The integration follows the existing pattern used by other chart callbacks on this page: + +- Add a new `html.Div(id="acheteur-distance-histogram")` placeholder in the layout +- Add a new callback with `Input("acheteur_data", "data")` that: + - Reconstructs `pl.LazyFrame(data)` from the store + - Calls `get_distance_histogram(lff)` + - Wraps the result in `make_card(...)` and returns it to the placeholder div + +### Titulaire page (`src/pages/titulaire.py`) + +Same pattern as acheteur: `dcc.Store` (`titulaire_data`) → new callback → `html.Div` placeholder. + +## Out of Scope + +- Filtering by distance range (could be a future filter on the observatoire page) +- Showing distance on a map or as a trend over time +- Bucket-based (named zone) grouping From 24db748b6a0a8184fd127c50ba63df729c955fdb Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:15:26 +0100 Subject: [PATCH 46/52] feat: add get_distance_histogram figure function Implement get_distance_histogram that creates a histogram of titulaire distances with logarithmic scale. Add 3 unit tests covering basic functionality, null handling, and edge cases. Also add DATA_SCHEMA_PATH to pytest env config. Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 3 ++- src/figures.py | 18 ++++++++++++++++++ tests/test_main.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2befbcf..b4c27e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ testpaths = [ ] env = [ "DATA_FILE_PARQUET_PATH=tests/test.parquet", - "DEVELOPMENT=true" + "DEVELOPMENT=true", + "DATA_SCHEMA_PATH=/home/colin/git/decp-processing/dist/schema.json" ] addopts = "-p no:warnings" diff --git a/src/figures.py b/src/figures.py index 451f24c..ea87874 100644 --- a/src/figures.py +++ b/src/figures.py @@ -579,6 +579,24 @@ def make_clusters_map(region: dict) -> dl.Map: return leaflet_map +def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: + dff = lff.select("titulaire_distance").drop_nulls().collect() + dff = dff.with_columns(pl.col("titulaire_distance").log(10)) + fig = px.histogram( + dff, + x="titulaire_distance", + nbins=50, + labels={"titulaire_distance": "Distance (km)"}, + ) + fig.update_xaxes( + tickvals=[0, 1, 2, 3, 4], + ticktext=["1", "10", "100", "1 000", "10 000"], + title_text="Distance (km)", + ) + fig.update_yaxes(title_text="Nombre de marchés") + return dcc.Graph(figure=fig) + + def make_card( title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4 ) -> dbc.Col: diff --git a/tests/test_main.py b/tests/test_main.py index 6a13762..a50d807 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -302,3 +302,36 @@ def test_009_observatoire_filter_persistence(dash_duo: DashComposite): assert acheteur_input.get_attribute("value") == "123", ( "URL param acheteur_id should override the value stored in localStorage" ) + + +def test_get_distance_histogram_returns_graph(): + import polars as pl + from dash import dcc + + from src.figures import get_distance_histogram + + lff = pl.LazyFrame({"titulaire_distance": [1, 10, 100, 500, 1000]}) + result = get_distance_histogram(lff) + assert isinstance(result, dcc.Graph) + + +def test_get_distance_histogram_handles_nulls(): + import polars as pl + from dash import dcc + + from src.figures import get_distance_histogram + + lff = pl.LazyFrame({"titulaire_distance": [None, None, 50]}) + result = get_distance_histogram(lff) + assert isinstance(result, dcc.Graph) + + +def test_get_distance_histogram_all_nulls(): + import polars as pl + from dash import dcc + + from src.figures import get_distance_histogram + + lff = pl.LazyFrame({"titulaire_distance": pl.Series([], dtype=pl.Int64)}) + result = get_distance_histogram(lff) + assert isinstance(result, dcc.Graph) From f2af09bb3581214f86070aaf15f26d28049cefbe Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:22:54 +0100 Subject: [PATCH 47/52] fix: use streaming collect and filter zero distances in get_distance_histogram --- src/figures.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/figures.py b/src/figures.py index ea87874..ea8d905 100644 --- a/src/figures.py +++ b/src/figures.py @@ -580,7 +580,12 @@ def make_clusters_map(region: dict) -> dl.Map: def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: - dff = lff.select("titulaire_distance").drop_nulls().collect() + dff = ( + lff.select("titulaire_distance") + .drop_nulls() + .filter(pl.col("titulaire_distance") > 0) + .collect(engine="streaming") + ) dff = dff.with_columns(pl.col("titulaire_distance").log(10)) fig = px.histogram( dff, From 44d2d7d2c1fa9fb2b5b7e83169a404a283ac9352 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:24:01 +0100 Subject: [PATCH 48/52] feat: add distance histogram card to observatoire dashboard Integrate the get_distance_histogram function into the observatoire dashboard to display buyer-contractor distance distribution on a logarithmic scale. Co-Authored-By: Claude Sonnet 4.6 --- src/pages/observatoire.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index f2bbe65..52ccbd4 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -19,6 +19,7 @@ from dash import ( from src.figures import ( get_barchart_sources, + get_distance_histogram, get_duplicate_matrix, get_geographic_maps, make_card, @@ -628,6 +629,15 @@ def udpate_dashboard_cards( ) ) + distance_histogram = get_distance_histogram(lff) + cards.append( + make_card( + title="Distance acheteur–titulaire", + subtitle="en nombre de marchés, échelle logarithmique", + fig=distance_histogram, + ) + ) + geographic_maps: list[dbc.Col] = get_geographic_maps(dff) other_cards = [] From b08d517f3667a764da345f6f737384ca8968c8ee Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:26:57 +0100 Subject: [PATCH 49/52] feat: add distance histogram to acheteur detail page --- src/pages/acheteur.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 76cd2d6..c6ffd9d 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -16,7 +16,13 @@ from dash import ( ) from src.callbacks import get_top_org_table -from src.figures import DataTable, make_column_picker, point_on_map +from src.figures import ( + DataTable, + get_distance_histogram, + make_card, + make_column_picker, + point_on_map, +) from src.utils import ( columns, df, @@ -140,6 +146,7 @@ layout = [ html.Div(className="marches_table", id="top10_titulaires"), ], ), + html.Div(id="acheteur-distance-histogram"), ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -476,3 +483,17 @@ def toggle_acheteur_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] + + +@callback( + Output("acheteur-distance-histogram", "children"), + Input("acheteur_data", "data"), +) +def update_acheteur_distance_histogram(data): + lff = pl.LazyFrame(data) + fig = get_distance_histogram(lff) + return make_card( + title="Distance acheteur–titulaire", + subtitle="en nombre de marchés, échelle logarithmique", + fig=fig, + ) From 858ab6c61ab1d65395e44e667127cba83ae5cb46 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:28:34 +0100 Subject: [PATCH 50/52] feat: add distance histogram to titulaire detail page --- src/pages/titulaire.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index ecee685..52ba25e 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -16,7 +16,13 @@ from dash import ( ) from src.callbacks import get_top_org_table -from src.figures import DataTable, make_column_picker, point_on_map +from src.figures import ( + DataTable, + get_distance_histogram, + make_card, + make_column_picker, + point_on_map, +) from src.utils import ( columns, df, @@ -140,6 +146,7 @@ layout = [ html.Div(className="marches_table", id="top10_acheteurs"), ], ), + html.Div(id="titulaire-distance-histogram"), ], ), # récupérer les données de l'acheteur sur l'api annuaire @@ -490,3 +497,21 @@ def toggle_titulaire_columns(click_open, click_close, is_open): ) def reset_view(n_clicks): return "", [] + + +@callback( + Output("titulaire-distance-histogram", "children"), + Input("titulaire_data", "data"), +) +def update_titulaire_distance_histogram(data): + lff = pl.LazyFrame(data) + if "titulaire_distance" in lff.collect_schema().names(): + lff = lff.with_columns( + pl.col("titulaire_distance").cast(pl.Float64, strict=False) + ) + fig = get_distance_histogram(lff) + return make_card( + title="Distance acheteur–titulaire", + subtitle="en nombre de marchés, échelle logarithmique", + fig=fig, + ) From 32aa8777971384d1dfe99f8cc7b37b5fbfd8982f Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:40:28 +0100 Subject: [PATCH 51/52] fix: guard against missing titulaire_distance column in get_distance_histogram --- src/figures.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/figures.py b/src/figures.py index ea8d905..14b624a 100644 --- a/src/figures.py +++ b/src/figures.py @@ -580,12 +580,15 @@ def make_clusters_map(region: dict) -> dl.Map: def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph: - dff = ( - lff.select("titulaire_distance") - .drop_nulls() - .filter(pl.col("titulaire_distance") > 0) - .collect(engine="streaming") - ) + if "titulaire_distance" not in lff.collect_schema().names(): + dff = pl.DataFrame({"titulaire_distance": pl.Series([], dtype=pl.Float64)}) + else: + dff = ( + lff.select("titulaire_distance") + .drop_nulls() + .filter(pl.col("titulaire_distance") > 0) + .collect(engine="streaming") + ) dff = dff.with_columns(pl.col("titulaire_distance").log(10)) fig = px.histogram( dff, From 8a61d3268aae4f4637e59b22e62336dea0716b69 Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Wed, 18 Mar 2026 22:53:17 +0100 Subject: [PATCH 52/52] =?UTF-8?q?fix:=20lecture=20du=20dataframe=20plus=20?= =?UTF-8?q?flexible=20dans=20la=20cr=C3=A9ation=20de=20l'histogram?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/pages/acheteur.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index c6ffd9d..a51846a 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -490,7 +490,7 @@ def reset_view(n_clicks): Input("acheteur_data", "data"), ) def update_acheteur_distance_histogram(data): - lff = pl.LazyFrame(data) + lff = pl.LazyFrame(data, strict=False, infer_schema_length=5000) fig = get_distance_histogram(lff) return make_card( title="Distance acheteur–titulaire",