colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

perf: paginer/agréger les pages acheteur et titulaire côté DuckDB

Browse Source 
Les pages acheteur/titulaire chargeaient l'intégralité des marchés d'une
organisation dans un dcc.Store côté client (jusqu'à 16k lignes pour les
plus gros acheteurs), envoyée sur le réseau à chaque interaction. Le
tableau, le top 10 et l'histogramme des distances récupèrent maintenant
leurs données via des requêtes DuckDB scopées (pagination/agrégation
poussées en SQL), sur le modèle déjà utilisé par la page tableau.

Corrige aussi le CLS des mêmes pages en réservant l'espace des
conteneurs remplis par callback (carte, top 10, histogramme).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Colin Maudry
2026-07-01 13:46:44 +02:00
parent 8515ba8d84
commit a2c20adb4e
5 changed files with 179 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/figures.py
+1 -1
View File
@@ -679,7 +679,7 @@ def get_distance_histogram(lff: pl.LazyFrame) -> dcc.Graph:
)
fig.update_layout(bargap=0)
fig.update_layout(margin=dict(r=10, t=10))
fig.update_layout(margin=dict(r=10, t=10), height=450)
fig.update_xaxes(
tickvals=[0, 1, 2, 3, 4],
ticktext=["1", "10", "100", "1 000", "10 000"],
src/pages/acheteur.py
+80 -50
View File
@@ -1,5 +1,4 @@
import datetime
from typing import Any
import dash_bootstrap_components as dbc
import polars as pl
@@ -15,7 +14,7 @@ from dash import (
register_page,
)
from src.db import query_marches, schema
from src.db import aggregate_marches, count_marches, query_marches, schema
from src.figures import (
DataTable,
get_distance_histogram,
@@ -48,6 +47,17 @@ def get_title(acheteur_id: str | None = None) -> str:
return "Marchés publics attribués | colibre"
def _acheteur_scope(pathname: str, ach_year: str | None) -> tuple[str, list]:
"""WHERE SQL scopant les requêtes à cet acheteur (et éventuellement une année)."""
acheteur_siret = pathname.split("/")[-1]
where_sql = "acheteur_id = ?"
params: list = [acheteur_siret]
if ach_year and ach_year != "Toutes les années":
where_sql += ' AND YEAR("dateNotification") = ?'
params.append(int(ach_year))
return where_sql, params
register_page(
__name__,
path_template="/acheteurs/<acheteur_id>",
@@ -75,7 +85,6 @@ DATATABLE = html.Div(
)
layout = [
dcc.Store(id="acheteur_data", storage_type="memory"),
dcc.Store(id="acheteur-hidden-columns", storage_type="local"),
dcc.Store(id="filter-cleanup-trigger-acheteur"),
dcc.Location(id="acheteur_url", refresh="callback-nav"),
@@ -159,6 +168,7 @@ layout = [
dbc.Col(
id="acheteur_map",
width=4,
style={"minHeight": "300px"},
),
],
),
@@ -168,8 +178,13 @@ layout = [
className="marches_table",
id="top10_titulaires",
width=8,
style={"minHeight": "420px"},
),
dbc.Col(
id="acheteur-distance-histogram",
width=4,
style={"minHeight": "450px"},
),
dbc.Col(id="acheteur-distance-histogram", width=4),
],
),
],
@@ -310,48 +325,38 @@ def update_acheteur_infos(url):
Output(
component_id="acheteur_titulaires_differents", component_property="children"
),
Input(component_id="acheteur_data", component_property="data"),
Input(component_id="acheteur_url", component_property="pathname"),
Input(component_id="acheteur_year", component_property="value"),
)
def update_acheteur_stats(data):
dff = pl.DataFrame(data, strict=False, infer_schema_length=5000)
if dff.height == 0:
dff = pl.DataFrame(schema=schema)
df_marches = dff.unique("id")
nb_marches = format_number(df_marches.height)
# somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item()))
marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"]
# + ", pour un total de ", html.Strong(somme_marches + " €")]
del df_marches
def update_acheteur_stats(pathname, ach_year):
where_sql, params = _acheteur_scope(pathname, ach_year)
agg = aggregate_marches(
"COUNT(*) AS n, COUNT(DISTINCT titulaire_id) AS nb_titulaires",
where_sql,
params,
)
nb_marches = format_number(int(agg["n"][0])) if agg.height else "0"
nb_titulaires = format_number(int(agg["nb_titulaires"][0])) if agg.height else "0"
nb_titulaires = dff.unique("titulaire_id").height
marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"]
nb_titulaires = [
html.Strong(format_number(nb_titulaires)),
html.Strong(nb_titulaires),
" titulaires (SIRET) différents",
]
del dff
return marches_attribues, nb_titulaires
@callback(
Output(component_id="acheteur_data", component_property="data"),
Output("btn-download-data-acheteur", "disabled"),
Output("btn-download-data-acheteur", "children"),
Output("btn-download-data-acheteur", "title"),
Input(component_id="acheteur_url", component_property="pathname"),
Input(component_id="acheteur_year", component_property="value"),
)
def get_acheteur_marches_data(url, ach_year: str) -> tuple:
acheteur_siret = url.split("/")[-1]
lff = query_marches("acheteur_id = ?", (acheteur_siret,)).lazy()
if ach_year and ach_year != "Toutes les années":
ach_year = int(ach_year)
lff = lff.filter(pl.col("dateNotification").dt.year() == ach_year)
lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True)
dff: pl.DataFrame = lff.collect(engine="streaming")
download_disabled, download_text, download_title = get_button_properties(dff.height)
data = dff.to_dicts()
return data, download_disabled, download_text, download_title
def update_download_button_acheteur(pathname, ach_year):
where_sql, params = _acheteur_scope(pathname, ach_year)
return get_button_properties(count_marches(where_sql, params))
@callback(
@@ -364,8 +369,8 @@ def get_acheteur_marches_data(url, ach_year: str) -> tuple:
Output("btn-download-filtered-data-acheteur", "children"),
Output("btn-download-filtered-data-acheteur", "title"),
Output("filter-cleanup-trigger-acheteur", "data"),
Input("acheteur_url", "href"),
Input("acheteur_data", "data"),
Input("acheteur_url", "pathname"),
Input("acheteur_year", "value"),
Input("acheteur_datatable", "page_current"),
Input("acheteur_datatable", "page_size"),
Input("acheteur_datatable", "filter_query"),
@@ -373,37 +378,59 @@ def get_acheteur_marches_data(url, ach_year: str) -> tuple:
State("acheteur_datatable", "data_timestamp"),
)
def get_last_marches_data(
href, data, page_current, page_size, filter_query, sort_by, data_timestamp
pathname,
ach_year,
page_current,
page_size,
filter_query,
sort_by,
data_timestamp,
) -> tuple:
where_sql, params = _acheteur_scope(pathname, ach_year)
return prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, "acheteur"
None,
data_timestamp,
filter_query,
page_current,
page_size,
sort_by,
"acheteur",
base_where_sql=where_sql,
base_params=params,
)
@callback(
Output(component_id="top10_titulaires", component_property="children"),
Input(component_id="acheteur_data", component_property="data"),
Input(component_id="acheteur_url", component_property="pathname"),
Input(component_id="acheteur_year", component_property="value"),
)
def get_top_titulaires(pathname, ach_year):
where_sql, params = _acheteur_scope(pathname, ach_year)
table = get_top_org_table(
query_marches(where_sql, params).lazy(), "titulaire", ["titulaire_distance"]
)
def get_top_titulaires(data):
table = get_top_org_table(data, "titulaire", ["titulaire_distance"])
return make_card(fig=table, title="Top titulaires", lg=12, xl=12)
@callback(
Output("download-data-acheteur", "data"),
Input("btn-download-data-acheteur", "n_clicks"),
State(component_id="acheteur_data", component_property="data"),
State(component_id="acheteur_nom", component_property="children"),
State(component_id="acheteur_url", component_property="pathname"),
State(component_id="acheteur_year", component_property="value"),
State(component_id="acheteur_nom", component_property="children"),
prevent_initial_call=True,
)
def download_acheteur_data(
n_clicks,
data: list[dict[str, Any]],
acheteur_nom: str,
pathname: str,
annee: str,
acheteur_nom: str,
):
df_to_download = pl.DataFrame(data)
where_sql, params = _acheteur_scope(pathname, annee)
df_to_download = query_marches(
where_sql, params, order_by='"dateNotification" DESC, uid DESC'
)
def to_bytes(buffer):
write_styled_excel(
@@ -418,8 +445,9 @@ def download_acheteur_data(
@callback(
Output("acheteur-download-filtered-data", "data"),
State("acheteur_data", "data"),
Input("btn-download-filtered-data-acheteur", "n_clicks"),
State("acheteur_url", "pathname"),
State("acheteur_year", "value"),
State("acheteur_nom", "children"),
State("acheteur_datatable", "filter_query"),
State("acheteur_datatable", "sort_by"),
@@ -427,16 +455,16 @@ def download_acheteur_data(
prevent_initial_call=True,
)
def download_filtered_acheteur_data(
data,
n_clicks,
pathname,
ach_year,
acheteur_nom,
filter_query,
sort_by,
hidden_columns: list | None = None,
):
lff: pl.LazyFrame = pl.LazyFrame(
data
) # start from the full acheteur data, not from paginated table data
where_sql, params = _acheteur_scope(pathname, ach_year)
lff: pl.LazyFrame = query_marches(where_sql, params).lazy()
# Les colonnes masquées sont supprimées
if hidden_columns:
@@ -535,10 +563,12 @@ def reset_view(n_clicks):
@callback(
Output("acheteur-distance-histogram", "children"),
Input("acheteur_data", "data"),
Input("acheteur_url", "pathname"),
Input("acheteur_year", "value"),
)
def update_acheteur_distance_histogram(data):
lff = pl.LazyFrame(data, strict=False, infer_schema_length=5000)
def update_acheteur_distance_histogram(pathname, ach_year):
where_sql, params = _acheteur_scope(pathname, ach_year)
lff = query_marches(where_sql, params).lazy()
fig = get_distance_histogram(lff)
return make_card(
title="Distance acheteur–titulaire",
src/pages/titulaire.py
+73 -54
View File
@@ -1,5 +1,4 @@
import datetime
from typing import Any
import dash_bootstrap_components as dbc
import polars as pl
@@ -15,7 +14,7 @@ from dash import (
register_page,
)
from src.db import query_marches, schema
from src.db import aggregate_marches, count_marches, query_marches, schema
from src.figures import (
DataTable,
get_distance_histogram,
@@ -47,6 +46,17 @@ def get_title(titulaire_id: str = None) -> str:
return "Marchés publics remportés | colibre"
def _titulaire_scope(pathname: str, titulaire_year: str | None) -> tuple[str, list]:
"""WHERE SQL scopant les requêtes à ce titulaire (et éventuellement une année)."""
titulaire_siret = pathname.split("/")[-1]
where_sql = "titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'"
params: list = [titulaire_siret]
if titulaire_year and titulaire_year != "Toutes les années":
where_sql += ' AND YEAR("dateNotification") = ?'
params.append(int(titulaire_year))
return where_sql, params
register_page(
__name__,
path_template="/titulaires/<titulaire_id>",
@@ -74,7 +84,6 @@ DATATABLE = html.Div(
)
layout = [
dcc.Store(id="titulaire_data", storage_type="memory"),
dcc.Store(id="titulaire-hidden-columns", storage_type="local"),
dcc.Store(id="filter-cleanup-trigger-titulaire"),
dcc.Location(id="titulaire_url", refresh="callback-nav"),
@@ -167,6 +176,7 @@ layout = [
dbc.Col(
id="titulaire_map",
width=4,
style={"minHeight": "300px"},
),
],
),
@@ -179,12 +189,17 @@ layout = [
html.Div(
className="marches_table",
id="top10_acheteurs",
style={"minHeight": "420px"},
),
],
),
width=8,
),
dbc.Col(id="titulaire-distance-histogram", width=4),
dbc.Col(
id="titulaire-distance-histogram",
width=4,
style={"minHeight": "450px"},
),
],
),
],
@@ -334,26 +349,25 @@ def update_titulaire_infos(url):
Output(
component_id="titulaire_acheteurs_differents", component_property="children"
),
Input(component_id="titulaire_data", component_property="data"),
Input(component_id="titulaire_url", component_property="pathname"),
Input(component_id="titulaire_year", component_property="value"),
)
def update_titulaire_stats(data):
dff = pl.DataFrame(data, strict=False, infer_schema_length=5000)
if dff.height == 0:
nb_marches = 0
nb_acheteurs = 0
else:
df_marches = dff.unique("uid")
nb_marches = format_number(df_marches.height)
nb_acheteurs = dff.unique("acheteur_id").height
def update_titulaire_stats(pathname, titulaire_year):
where_sql, params = _titulaire_scope(pathname, titulaire_year)
agg = aggregate_marches(
"COUNT(DISTINCT uid) AS n, COUNT(DISTINCT acheteur_id) AS nb_acheteurs",
where_sql,
params,
)
nb_marches = format_number(int(agg["n"][0])) if agg.height else "0"
nb_acheteurs = format_number(int(agg["nb_acheteurs"][0])) if agg.height else "0"
texte_marches_remportes = [
html.Strong(nb_marches),
" marchés et accord-cadres remportés",
]
# + ", pour un total de ", html.Strong(somme_marches + " €")]
texte_nb_acheteurs = [
html.Strong(format_number(nb_acheteurs)),
html.Strong(nb_acheteurs),
" acheteurs (SIRET) différents",
]
@@ -361,29 +375,15 @@ def update_titulaire_stats(data):
@callback(
Output(component_id="titulaire_data", component_property="data"),
Output("btn-download-data-titulaire", "disabled"),
Output("btn-download-data-titulaire", "children"),
Output("btn-download-data-titulaire", "title"),
Input(component_id="titulaire_url", component_property="pathname"),
Input(component_id="titulaire_year", component_property="value"),
)
def get_titulaire_marches_data(url, titulaire_year: str) -> tuple:
titulaire_siret = url.split("/")[-1]
lff = query_marches(
"titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'",
(titulaire_siret,),
).lazy()
if titulaire_year and titulaire_year != "Toutes les années":
lff = lff.filter(
pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year)
)
lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True)
lff = lff.fill_null("")
dff: pl.DataFrame = lff.collect(engine="streaming")
download_disabled, download_text, download_title = get_button_properties(dff.height)
data = dff.to_dicts()
return data, download_disabled, download_text, download_title
def update_download_button_titulaire(pathname, titulaire_year):
where_sql, params = _titulaire_scope(pathname, titulaire_year)
return get_button_properties(count_marches(where_sql, params))
@callback(
@@ -396,8 +396,8 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple:
Output("btn-download-filtered-data-titulaire", "children"),
Output("btn-download-filtered-data-titulaire", "title"),
Output("filter-cleanup-trigger-titulaire", "data"),
Input(component_id="titulaire_url", component_property="href"),
Input("titulaire_data", "data"),
Input("titulaire_url", "pathname"),
Input("titulaire_year", "value"),
Input("titulaire_datatable", "page_current"),
Input("titulaire_datatable", "page_size"),
Input("titulaire_datatable", "filter_query"),
@@ -405,42 +405,58 @@ def get_titulaire_marches_data(url, titulaire_year: str) -> tuple:
State("titulaire_datatable", "data_timestamp"),
)
def get_last_marches_data(
href, data, page_current, page_size, filter_query, sort_by, data_timestamp
pathname,
titulaire_year,
page_current,
page_size,
filter_query,
sort_by,
data_timestamp,
) -> list[dict]:
where_sql, params = _titulaire_scope(pathname, titulaire_year)
return prepare_table_data(
data,
None,
data_timestamp,
filter_query,
page_current,
page_size,
sort_by,
"titulaire",
base_where_sql=where_sql,
base_params=params,
)
@callback(
Output(component_id="top10_acheteurs", component_property="children"),
Input(component_id="titulaire_data", component_property="data"),
Input(component_id="titulaire_url", component_property="pathname"),
Input(component_id="titulaire_year", component_property="value"),
)
def get_top_acheteurs(pathname, titulaire_year):
where_sql, params = _titulaire_scope(pathname, titulaire_year)
return get_top_org_table(
query_marches(where_sql, params).lazy(), "acheteur", ["titulaire_distance"]
)
def get_top_acheteurs(data):
return get_top_org_table(data, "acheteur", ["titulaire_distance"])
@callback(
Output("download-data-titulaire", "data"),
Input("btn-download-data-titulaire", "n_clicks"),
State(component_id="titulaire_data", component_property="data"),
State(component_id="titulaire_nom", component_property="children"),
State(component_id="titulaire_url", component_property="pathname"),
State(component_id="titulaire_year", component_property="value"),
State(component_id="titulaire_nom", component_property="children"),
prevent_initial_call=True,
)
def download_titulaire_data(
n_clicks,
data: list[dict[str, Any]],
titulaire_nom: str,
pathname: str,
annee: str,
titulaire_nom: str,
):
df_to_download = pl.DataFrame(data)
where_sql, params = _titulaire_scope(pathname, annee)
df_to_download = query_marches(
where_sql, params, order_by='"dateNotification" DESC, uid DESC'
).fill_null("")
def to_bytes(buffer):
write_styled_excel(
@@ -455,8 +471,9 @@ def download_titulaire_data(
@callback(
Output("titulaire-download-filtered-data", "data"),
State("titulaire_data", "data"),
Input("btn-download-filtered-data-titulaire", "n_clicks"),
State("titulaire_url", "pathname"),
State("titulaire_year", "value"),
State("titulaire_nom", "children"),
State("titulaire_datatable", "filter_query"),
State("titulaire_datatable", "sort_by"),
@@ -464,16 +481,16 @@ def download_titulaire_data(
prevent_initial_call=True,
)
def download_filtered_titulaire_data(
data,
n_clicks,
pathname,
titulaire_year,
titulaire_nom,
filter_query,
sort_by,
hidden_columns: list | None = None,
):
lff: pl.LazyFrame = pl.LazyFrame(
data
) # start from the full titulaire data, not from paginated table data
where_sql, params = _titulaire_scope(pathname, titulaire_year)
lff: pl.LazyFrame = query_marches(where_sql, params).lazy()
# Les colonnes masquées sont supprimées
if hidden_columns:
@@ -572,10 +589,12 @@ def reset_view(n_clicks):
@callback(
Output("titulaire-distance-histogram", "children"),
Input("titulaire_data", "data"),
Input("titulaire_url", "pathname"),
Input("titulaire_year", "value"),
)
def update_titulaire_distance_histogram(data):
lff = pl.LazyFrame(data)
def update_titulaire_distance_histogram(pathname, titulaire_year):
where_sql, params = _titulaire_scope(pathname, titulaire_year)
lff = query_marches(where_sql, params).lazy()
if "titulaire_distance" in lff.collect_schema().names():
lff = lff.with_columns(
pl.col("titulaire_distance").cast(pl.Float64, strict=False)
src/utils/table.py
+23 -3
View File
@@ -427,9 +427,14 @@ def _fetch_page_sql(
sort_by_key: tuple,
page_current: int,
page_size: int,
base_where_sql: str = "TRUE",
base_params: tuple = (),
) -> tuple[pl.DataFrame, int, int]:
"""Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule.
`base_where_sql`/`base_params` permettent de scoper la requête (ex : un
acheteur ou un titulaire précis) en plus du filtre saisi dans la table.
Retourne (page_dataframe_post_traitée, total_count, total_unique_count).
"""
# Import local pour éviter une dépendance circulaire
@@ -441,7 +446,9 @@ def _fetch_page_sql(
f"page={page_current} size={page_size}"
)
where_sql, params = filter_query_to_sql(filter_query or "", schema)
filter_where_sql, filter_params = filter_query_to_sql(filter_query or "", schema)
where_sql = f"({base_where_sql}) AND ({filter_where_sql})"
params = [*base_params, *filter_params]
sort_by_dash = [
{"column_id": col, "direction": direction} for col, direction in sort_by_key
@@ -464,7 +471,15 @@ def _fetch_page_sql(
def prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
data,
data_timestamp,
filter_query,
page_current,
page_size,
sort_by,
source_table,
base_where_sql: str = "TRUE",
base_params: tuple = (),
):
"""
Fonction de préparation des données pour les datatables, afin de permettre une gestion fine des logiques,
@@ -476,6 +491,8 @@ def prepare_table_data(
:param page_size:
:param sort_by:
:param source_table:
:param base_where_sql: scope SQL additionnel (ex : un acheteur/titulaire précis)
:param base_params: paramètres liés à base_where_sql
:return:
"""
logger.debug(" + + + + + + + + + + + + + + + + + + ")
@@ -486,13 +503,16 @@ def prepare_table_data(
trigger_cleanup = no_update if source_table == "tableau" else str(uuid.uuid4())
if data is None:
# Probablement car il s'agit de la page Tableau
# Chemin rapide SQL : tableau, acheteur, titulaire (scope éventuel via
# base_where_sql/base_params)
sort_by_key = normalize_sort_by(sort_by)
dff, height, total_unique = _fetch_page_sql(
filter_query=filter_query,
sort_by_key=sort_by_key,
page_current=page_current,
page_size=page_size,
base_where_sql=base_where_sql,
base_params=tuple(base_params),
)
else:
if isinstance(data, list):
tests/test_main.py
+2 -11
View File
@@ -98,12 +98,10 @@ def test_003_tableau_download(dash_duo: DashComposite):
# Juste pour instancier l'app
print(app.server.name)
dicts = pl.read_parquet("tests/test.parquet").to_dicts()
outputs = [
download_data(1, "", [], None),
download_acheteur_data(1, dicts, "123", "2025"),
download_titulaire_data(1, dicts, "345", "2025"),
download_acheteur_data(1, "/acheteurs/123", "2025", "ACHETEUR 1"),
download_titulaire_data(1, "/titulaires/345", "2025", "TITULAIRE 1"),
]
for output in outputs:
assert isinstance(output, dict)
@@ -118,8 +116,6 @@ def test_003_tableau_download(dash_duo: DashComposite):
def test_004_add_links_observatoire_acheteur():
import polars as pl
from src.utils.table import add_links
dff = pl.DataFrame(
@@ -143,8 +139,6 @@ def test_004_add_links_observatoire_acheteur():
def test_005_add_links_observatoire_titulaire():
import polars as pl
from src.utils.table import add_links
dff = pl.DataFrame(
@@ -323,7 +317,6 @@ def test_011_observatoire_multi_param_url(dash_duo: DashComposite):
def test_012_get_distance_histogram_returns_graph():
import polars as pl
from dash import dcc
from src.figures import get_distance_histogram
@@ -334,7 +327,6 @@ def test_012_get_distance_histogram_returns_graph():
def test_013_get_distance_histogram_handles_nulls():
import polars as pl
from dash import dcc
from src.figures import get_distance_histogram
@@ -345,7 +337,6 @@ def test_013_get_distance_histogram_handles_nulls():
def test_014_get_distance_histogram_all_nulls():
import polars as pl
from dash import dcc
from src.figures import get_distance_histogram