From 1e67d329d03fa65cc1f032f081e0ac929c79b0bb Mon Sep 17 00:00:00 2001 From: Colin Maudry Date: Tue, 21 Apr 2026 23:53:45 +0200 Subject: [PATCH] perf(tableau): pousser filtre/tri/pagination/comptage dans DuckDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remplace le chemin lent de prepare_table_data (chargement de toutes les lignes depuis DuckDB puis filtrage/post-traitement Polars avant slice) par _fetch_page_sql qui pousse filtre, tri, pagination et comptage dans DuckDB via filter_query_to_sql / sort_by_to_sql, puis post-traite uniquement la page de 20 lignes. Supprime _load_filter_sort_postprocess (plus utilisé). Met à jour les tests test_table.py en supprimant les tests associés et en ajoutant des tests dédiés pour _fetch_page_sql. Corrige le fixture flask_app pour utiliser src.utils.cache (même instance que le module) afin que @cache.memoize() fonctionne. Co-Authored-By: Claude Sonnet 4.6 --- src/utils/table.py | 88 +++++++++++++++++--------- tests/test_table.py | 151 ++++++++++++++++++++------------------------ 2 files changed, 124 insertions(+), 115 deletions(-) diff --git a/src/utils/table.py b/src/utils/table.py index 2c0d2ea..9e47142 100644 --- a/src/utils/table.py +++ b/src/utils/table.py @@ -5,7 +5,7 @@ import polars as pl from dash import no_update from polars import selectors as cs -from src.db import query_marches, schema +from src.db import count_marches, count_unique_marches, query_marches, schema from src.utils import logger from src.utils.cache import cache from src.utils.data import DATA_SCHEMA @@ -376,28 +376,6 @@ def get_default_hidden_columns(page): return hidden_columns -@cache.memoize() -def _load_filter_sort_postprocess(filter_query, sort_by_key): - logger.debug( - f"Cache miss — recomputing for filter={filter_query!r} sort={sort_by_key!r}" - ) - - lff: pl.LazyFrame = query_marches().lazy() - - if filter_query: - lff = filter_table_data(lff, filter_query) - - if sort_by_key: - sort_by = [ - {"column_id": col, "direction": direction} for col, direction in sort_by_key - ] - lff = sort_table_data(lff, sort_by) - - dff = table_postprocess(lff) - - return dff - - def table_postprocess(lff) -> pl.DataFrame: lff = lff.cast(pl.String) lff = lff.fill_null("") @@ -426,6 +404,48 @@ def postprocess_page(dff: pl.DataFrame) -> pl.DataFrame: return dff +@cache.memoize() +def _fetch_page_sql( + filter_query: str | None, + sort_by_key: tuple, + page_current: int, + page_size: int, +) -> tuple[pl.DataFrame, int, int]: + """Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule. + + Retourne (page_dataframe_post_traitée, total_count, total_unique_count). + """ + # Import local pour éviter une dépendance circulaire + # (src.utils.table_sql importe split_filter_part depuis src.utils.table). + from src.utils.table_sql import filter_query_to_sql, sort_by_to_sql + + logger.debug( + f"Cache miss SQL — filter={filter_query!r} sort={sort_by_key!r} " + f"page={page_current} size={page_size}" + ) + + where_sql, params = filter_query_to_sql(filter_query or "", schema) + + sort_by_dash = [ + {"column_id": col, "direction": direction} for col, direction in sort_by_key + ] + order_by = sort_by_to_sql(sort_by_dash, schema) or None + + total = count_marches(where_sql, params) + total_unique = count_unique_marches(where_sql, params) + + page = query_marches( + where_sql=where_sql, + params=params, + order_by=order_by, + limit=page_size, + offset=page_current * page_size, + ) + + page = postprocess_page(page) + return page, total, total_unique + + def prepare_table_data( data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table ): @@ -450,10 +470,16 @@ def prepare_table_data( if data is None: sort_by_key = normalize_sort_by(sort_by) - dff: pl.DataFrame = _load_filter_sort_postprocess( - filter_query=filter_query, sort_by_key=sort_by_key + dff, total, total_unique = _fetch_page_sql( + filter_query=filter_query, + sort_by_key=sort_by_key, + page_current=page_current, + page_size=page_size, ) + height = total + already_paginated = True else: + already_paginated = False if isinstance(data, list): lff: pl.LazyFrame = pl.LazyFrame( data, strict=False, infer_schema_length=5000 @@ -470,19 +496,19 @@ def prepare_table_data( lff = sort_table_data(lff, sort_by) dff: pl.DataFrame = table_postprocess(lff) - - height = dff.height + height = dff.height + total_unique = dff.select("uid").unique().height if "uid" in dff.columns else 0 if height > 0: nb_rows = ( - f"{format_number(height)} lignes " - f"({format_number(dff.select('uid').unique().height)} marchés)" + f"{format_number(height)} lignes ({format_number(total_unique)} marchés)" ) else: nb_rows = "0 lignes (0 marchés)" - start_row = page_current * page_size - dff = dff.slice(start_row, page_size) + if not already_paginated: + start_row = page_current * page_size + dff = dff.slice(start_row, page_size) table_columns, tooltip = setup_table_columns(dff) diff --git a/tests/test_table.py b/tests/test_table.py index c47f967..65a9f52 100644 --- a/tests/test_table.py +++ b/tests/test_table.py @@ -83,7 +83,7 @@ def flask_app(): """Minimal Flask app with SimpleCache so @cache.memoize() works in tests.""" from flask import Flask - from utils.cache import cache + from src.utils.cache import cache app = Flask(__name__) cache.init_app(app, config={"CACHE_TYPE": "SimpleCache"}) @@ -95,7 +95,7 @@ def reset_cache(flask_app): """Ensure the flask-caching backend is empty between tests so that cache-hit assertions are meaningful. Falls back to no-op when no Flask app context is active (NullCache).""" - from utils.cache import cache + from src.utils.cache import cache with flask_app.app_context(): try: @@ -106,60 +106,9 @@ def reset_cache(flask_app): yield -def test_load_filter_sort_postprocess_returns_dataframe( - flask_app, monkeypatch, sample_lff -): +def test_prepare_table_data_returns_expected_tuple(flask_app): from src.utils import table - monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect()) - - with flask_app.app_context(): - df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=()) - - assert isinstance(df, pl.DataFrame) - assert df.height == 1 - # All values must be strings after post-processing - for col in df.columns: - assert df.schema[col] == pl.String - - -def test_load_filter_sort_postprocess_applies_filter( - flask_app, monkeypatch, sample_lff -): - from src.utils import table - - monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect()) - - with flask_app.app_context(): - df = table._load_filter_sort_postprocess( - filter_query="{objet} icontains travaux", sort_by_key=() - ) - assert df.height == 1 - - df_empty = table._load_filter_sort_postprocess( - filter_query="{objet} icontains nonexistent", sort_by_key=() - ) - assert df_empty.height == 0 - - -def test_load_filter_sort_postprocess_adds_links(flask_app, monkeypatch, sample_lff): - from src.utils import table - - monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect()) - - with flask_app.app_context(): - df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=()) - # add_links injects an wrapper around uid, acheteur_nom, titulaire_nom - assert "= 1 -def test_prepare_table_data_cleanup_trigger_for_non_tableau( - monkeypatch, flask_app, sample_lff -): +def test_prepare_table_data_cleanup_trigger_for_non_tableau(flask_app): """Non-tableau pages still get a fresh uuid trigger, not no_update.""" from dash import no_update from src.utils import table - monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect()) - with flask_app.app_context(): result = table.prepare_table_data( data=None, @@ -289,11 +234,11 @@ def test_prepare_table_data_with_external_data_does_not_use_cache( sentinel["called"] = True raise AssertionError("Memoized helper must not be called when data is provided") - monkeypatch.setattr(table, "_load_filter_sort_postprocess", should_not_be_called) + monkeypatch.setattr(table, "_fetch_page_sql", should_not_be_called) with flask_app.app_context(): table.prepare_table_data( - data=sample_lff, # external LazyFrame + data=sample_lff, data_timestamp=0, filter_query=None, page_current=0, @@ -305,6 +250,44 @@ def test_prepare_table_data_with_external_data_does_not_use_cache( assert sentinel["called"] is False +def test_fetch_page_sql_respects_pagination(flask_app): + """New path: returns (page_dff, total_count, total_unique) via DuckDB.""" + from src.utils import table + + with flask_app.app_context(): + page, total, total_unique = table._fetch_page_sql( + filter_query=None, sort_by_key=(), page_current=0, page_size=5 + ) + assert page.height <= 5 + assert total >= page.height + assert isinstance(total_unique, int) + + +def test_fetch_page_sql_applies_filter(flask_app): + from src.utils import table + + with flask_app.app_context(): + page, total, total_unique = table._fetch_page_sql( + filter_query="{uid} icontains __ne_matche_rien__", + sort_by_key=(), + page_current=0, + page_size=20, + ) + assert total == 0 + assert page.height == 0 + + +def test_fetch_page_sql_post_processes_links(flask_app): + from src.utils import table + + with flask_app.app_context(): + page, _, _ = table._fetch_page_sql( + filter_query=None, sort_by_key=(), page_current=0, page_size=1 + ) + if page.height > 0: + assert "