perf(tableau): pousser filtre/tri/pagination/comptage dans DuckDB

Remplace le chemin lent de prepare_table_data (chargement de toutes les
lignes depuis DuckDB puis filtrage/post-traitement Polars avant slice)
par _fetch_page_sql qui pousse filtre, tri, pagination et comptage dans
DuckDB via filter_query_to_sql / sort_by_to_sql, puis post-traite
uniquement la page de 20 lignes.

Supprime _load_filter_sort_postprocess (plus utilisé). Met à jour les
tests test_table.py en supprimant les tests associés et en ajoutant
des tests dédiés pour _fetch_page_sql. Corrige le fixture flask_app
pour utiliser src.utils.cache (même instance que le module) afin que
@cache.memoize() fonctionne.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Colin Maudry
2026-04-21 23:53:45 +02:00
parent 4e25ff5c85
commit 1e67d329d0
2 changed files with 124 additions and 115 deletions
+57 -31
View File
@@ -5,7 +5,7 @@ import polars as pl
from dash import no_update
from polars import selectors as cs
from src.db import query_marches, schema
from src.db import count_marches, count_unique_marches, query_marches, schema
from src.utils import logger
from src.utils.cache import cache
from src.utils.data import DATA_SCHEMA
@@ -376,28 +376,6 @@ def get_default_hidden_columns(page):
return hidden_columns
@cache.memoize()
def _load_filter_sort_postprocess(filter_query, sort_by_key):
logger.debug(
f"Cache miss — recomputing for filter={filter_query!r} sort={sort_by_key!r}"
)
lff: pl.LazyFrame = query_marches().lazy()
if filter_query:
lff = filter_table_data(lff, filter_query)
if sort_by_key:
sort_by = [
{"column_id": col, "direction": direction} for col, direction in sort_by_key
]
lff = sort_table_data(lff, sort_by)
dff = table_postprocess(lff)
return dff
def table_postprocess(lff) -> pl.DataFrame:
lff = lff.cast(pl.String)
lff = lff.fill_null("")
@@ -426,6 +404,48 @@ def postprocess_page(dff: pl.DataFrame) -> pl.DataFrame:
return dff
@cache.memoize()
def _fetch_page_sql(
filter_query: str | None,
sort_by_key: tuple,
page_current: int,
page_size: int,
) -> tuple[pl.DataFrame, int, int]:
"""Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule.
Retourne (page_dataframe_post_traitée, total_count, total_unique_count).
"""
# Import local pour éviter une dépendance circulaire
# (src.utils.table_sql importe split_filter_part depuis src.utils.table).
from src.utils.table_sql import filter_query_to_sql, sort_by_to_sql
logger.debug(
f"Cache miss SQL — filter={filter_query!r} sort={sort_by_key!r} "
f"page={page_current} size={page_size}"
)
where_sql, params = filter_query_to_sql(filter_query or "", schema)
sort_by_dash = [
{"column_id": col, "direction": direction} for col, direction in sort_by_key
]
order_by = sort_by_to_sql(sort_by_dash, schema) or None
total = count_marches(where_sql, params)
total_unique = count_unique_marches(where_sql, params)
page = query_marches(
where_sql=where_sql,
params=params,
order_by=order_by,
limit=page_size,
offset=page_current * page_size,
)
page = postprocess_page(page)
return page, total, total_unique
def prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
):
@@ -450,10 +470,16 @@ def prepare_table_data(
if data is None:
sort_by_key = normalize_sort_by(sort_by)
dff: pl.DataFrame = _load_filter_sort_postprocess(
filter_query=filter_query, sort_by_key=sort_by_key
dff, total, total_unique = _fetch_page_sql(
filter_query=filter_query,
sort_by_key=sort_by_key,
page_current=page_current,
page_size=page_size,
)
height = total
already_paginated = True
else:
already_paginated = False
if isinstance(data, list):
lff: pl.LazyFrame = pl.LazyFrame(
data, strict=False, infer_schema_length=5000
@@ -470,19 +496,19 @@ def prepare_table_data(
lff = sort_table_data(lff, sort_by)
dff: pl.DataFrame = table_postprocess(lff)
height = dff.height
height = dff.height
total_unique = dff.select("uid").unique().height if "uid" in dff.columns else 0
if height > 0:
nb_rows = (
f"{format_number(height)} lignes "
f"({format_number(dff.select('uid').unique().height)} marchés)"
f"{format_number(height)} lignes ({format_number(total_unique)} marchés)"
)
else:
nb_rows = "0 lignes (0 marchés)"
start_row = page_current * page_size
dff = dff.slice(start_row, page_size)
if not already_paginated:
start_row = page_current * page_size
dff = dff.slice(start_row, page_size)
table_columns, tooltip = setup_table_columns(dff)