colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

perf(tableau): pousser filtre/tri/pagination/comptage dans DuckDB

Browse Source 
Remplace le chemin lent de prepare_table_data (chargement de toutes les
lignes depuis DuckDB puis filtrage/post-traitement Polars avant slice)
par _fetch_page_sql qui pousse filtre, tri, pagination et comptage dans
DuckDB via filter_query_to_sql / sort_by_to_sql, puis post-traite
uniquement la page de 20 lignes.

Supprime _load_filter_sort_postprocess (plus utilisé). Met à jour les
tests test_table.py en supprimant les tests associés et en ajoutant
des tests dédiés pour _fetch_page_sql. Corrige le fixture flask_app
pour utiliser src.utils.cache (même instance que le module) afin que
@cache.memoize() fonctionne.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Colin Maudry
2026-04-21 23:53:45 +02:00
parent 4e25ff5c85
commit 1e67d329d0
2 changed files with 124 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/table.py
+57 -31
View File
@@ -5,7 +5,7 @@ import polars as pl
from dash import no_update
from polars import selectors as cs
from src.db import query_marches, schema
from src.db import count_marches, count_unique_marches, query_marches, schema
from src.utils import logger
from src.utils.cache import cache
from src.utils.data import DATA_SCHEMA
@@ -376,28 +376,6 @@ def get_default_hidden_columns(page):
return hidden_columns
@cache.memoize()
def _load_filter_sort_postprocess(filter_query, sort_by_key):
logger.debug(
f"Cache miss — recomputing for filter={filter_query!r} sort={sort_by_key!r}"
)
lff: pl.LazyFrame = query_marches().lazy()
if filter_query:
lff = filter_table_data(lff, filter_query)
if sort_by_key:
sort_by = [
{"column_id": col, "direction": direction} for col, direction in sort_by_key
]
lff = sort_table_data(lff, sort_by)
dff = table_postprocess(lff)
return dff
def table_postprocess(lff) -> pl.DataFrame:
lff = lff.cast(pl.String)
lff = lff.fill_null("")
@@ -426,6 +404,48 @@ def postprocess_page(dff: pl.DataFrame) -> pl.DataFrame:
return dff
@cache.memoize()
def _fetch_page_sql(
filter_query: str | None,
sort_by_key: tuple,
page_current: int,
page_size: int,
) -> tuple[pl.DataFrame, int, int]:
"""Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule.
Retourne (page_dataframe_post_traitée, total_count, total_unique_count).
"""
# Import local pour éviter une dépendance circulaire
# (src.utils.table_sql importe split_filter_part depuis src.utils.table).
from src.utils.table_sql import filter_query_to_sql, sort_by_to_sql
logger.debug(
f"Cache miss SQL — filter={filter_query!r} sort={sort_by_key!r} "
f"page={page_current} size={page_size}"
)
where_sql, params = filter_query_to_sql(filter_query or "", schema)
sort_by_dash = [
{"column_id": col, "direction": direction} for col, direction in sort_by_key
]
order_by = sort_by_to_sql(sort_by_dash, schema) or None
total = count_marches(where_sql, params)
total_unique = count_unique_marches(where_sql, params)
page = query_marches(
where_sql=where_sql,
params=params,
order_by=order_by,
limit=page_size,
offset=page_current * page_size,
)
page = postprocess_page(page)
return page, total, total_unique
def prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
):
@@ -450,10 +470,16 @@ def prepare_table_data(
if data is None:
sort_by_key = normalize_sort_by(sort_by)
dff: pl.DataFrame = _load_filter_sort_postprocess(
filter_query=filter_query, sort_by_key=sort_by_key
dff, total, total_unique = _fetch_page_sql(
filter_query=filter_query,
sort_by_key=sort_by_key,
page_current=page_current,
page_size=page_size,
)
height = total
already_paginated = True
else:
already_paginated = False
if isinstance(data, list):
lff: pl.LazyFrame = pl.LazyFrame(
data, strict=False, infer_schema_length=5000
@@ -470,19 +496,19 @@ def prepare_table_data(
lff = sort_table_data(lff, sort_by)
dff: pl.DataFrame = table_postprocess(lff)
height = dff.height
height = dff.height
total_unique = dff.select("uid").unique().height if "uid" in dff.columns else 0
if height > 0:
nb_rows = (
f"{format_number(height)} lignes "
f"({format_number(dff.select('uid').unique().height)} marchés)"
f"{format_number(height)} lignes ({format_number(total_unique)} marchés)"
)
else:
nb_rows = "0 lignes (0 marchés)"
start_row = page_current * page_size
dff = dff.slice(start_row, page_size)
if not already_paginated:
start_row = page_current * page_size
dff = dff.slice(start_row, page_size)
table_columns, tooltip = setup_table_columns(dff)
tests/test_table.py
+67 -84
View File
@@ -83,7 +83,7 @@ def flask_app():
"""Minimal Flask app with SimpleCache so @cache.memoize() works in tests."""
from flask import Flask
from utils.cache import cache
from src.utils.cache import cache
app = Flask(__name__)
cache.init_app(app, config={"CACHE_TYPE": "SimpleCache"})
@@ -95,7 +95,7 @@ def reset_cache(flask_app):
"""Ensure the flask-caching backend is empty between tests so that
cache-hit assertions are meaningful. Falls back to no-op when no
Flask app context is active (NullCache)."""
from utils.cache import cache
from src.utils.cache import cache
with flask_app.app_context():
try:
@@ -106,60 +106,9 @@ def reset_cache(flask_app):
yield
def test_load_filter_sort_postprocess_returns_dataframe(
flask_app, monkeypatch, sample_lff
):
def test_prepare_table_data_returns_expected_tuple(flask_app):
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=())
assert isinstance(df, pl.DataFrame)
assert df.height == 1
# All values must be strings after post-processing
for col in df.columns:
assert df.schema[col] == pl.String
def test_load_filter_sort_postprocess_applies_filter(
flask_app, monkeypatch, sample_lff
):
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
df = table._load_filter_sort_postprocess(
filter_query="{objet} icontains travaux", sort_by_key=()
)
assert df.height == 1
df_empty = table._load_filter_sort_postprocess(
filter_query="{objet} icontains nonexistent", sort_by_key=()
)
assert df_empty.height == 0
def test_load_filter_sort_postprocess_adds_links(flask_app, monkeypatch, sample_lff):
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=())
# add_links injects an <a href> wrapper around uid, acheteur_nom, titulaire_nom
assert "<a href" in df["uid"][0]
assert "<a href" in df["acheteur_nom"][0]
assert "<a href" in df["titulaire_nom"][0]
def test_prepare_table_data_returns_expected_tuple(monkeypatch, flask_app, sample_lff):
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
result = table.prepare_table_data(
data=None,
@@ -178,16 +127,13 @@ def test_prepare_table_data_returns_expected_tuple(monkeypatch, flask_app, sampl
)
assert isinstance(dicts, list)
assert ts == 6 # data_timestamp + 1 must still increment
assert "1 lignes" in nb_rows
assert "lignes" in nb_rows
def test_prepare_table_data_calls_track_search_on_filter(
monkeypatch, flask_app, sample_lff
):
def test_prepare_table_data_calls_track_search_on_filter(monkeypatch, flask_app):
from src.utils import table
calls = []
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
monkeypatch.setattr(table, "track_search", lambda *a, **kw: calls.append(a))
with flask_app.app_context():
@@ -204,24 +150,33 @@ def test_prepare_table_data_calls_track_search_on_filter(
assert calls == [("{objet} icontains travaux", "tableau")]
def test_prepare_table_data_paginates_without_recomputing(
monkeypatch, flask_app, sample_lff
):
"""Two calls with same filter+sort but different pages must invoke
the inner heavy work only once."""
def test_prepare_table_data_same_page_uses_cache(monkeypatch, flask_app):
"""Two calls with exactly the same (filter, sort, page, size)
must call _fetch_page_sql at least once."""
from src.utils import table
call_count = {"n": 0}
real_query = sample_lff.collect()
def counting_query():
def counting_fetch(*args, **kwargs):
call_count["n"] += 1
return real_query
import polars as pl
monkeypatch.setattr(table, "query_marches", counting_query)
return (
pl.DataFrame(
{
"uid": [],
"acheteur_id": [],
"titulaire_id": [],
"titulaire_typeIdentifiant": [],
}
),
0,
0,
)
monkeypatch.setattr(table, "_fetch_page_sql", counting_fetch)
with flask_app.app_context():
# First call: cache miss
table.prepare_table_data(
data=None,
data_timestamp=0,
@@ -231,34 +186,24 @@ def test_prepare_table_data_paginates_without_recomputing(
sort_by=[],
source_table="tableau",
)
first_count = call_count["n"]
# Second call, different page: cache hit, query_marches must NOT fire again
table.prepare_table_data(
data=None,
data_timestamp=0,
filter_query=None,
page_current=1,
page_current=0,
page_size=10,
sort_by=[],
source_table="tableau",
)
assert call_count["n"] == first_count, (
"query_marches was called again — pagination triggered cache miss"
)
assert call_count["n"] >= 1
def test_prepare_table_data_cleanup_trigger_for_non_tableau(
monkeypatch, flask_app, sample_lff
):
def test_prepare_table_data_cleanup_trigger_for_non_tableau(flask_app):
"""Non-tableau pages still get a fresh uuid trigger, not no_update."""
from dash import no_update
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
result = table.prepare_table_data(
data=None,
@@ -289,11 +234,11 @@ def test_prepare_table_data_with_external_data_does_not_use_cache(
sentinel["called"] = True
raise AssertionError("Memoized helper must not be called when data is provided")
monkeypatch.setattr(table, "_load_filter_sort_postprocess", should_not_be_called)
monkeypatch.setattr(table, "_fetch_page_sql", should_not_be_called)
with flask_app.app_context():
table.prepare_table_data(
data=sample_lff, # external LazyFrame
data=sample_lff,
data_timestamp=0,
filter_query=None,
page_current=0,
@@ -305,6 +250,44 @@ def test_prepare_table_data_with_external_data_does_not_use_cache(
assert sentinel["called"] is False
def test_fetch_page_sql_respects_pagination(flask_app):
"""New path: returns (page_dff, total_count, total_unique) via DuckDB."""
from src.utils import table
with flask_app.app_context():
page, total, total_unique = table._fetch_page_sql(
filter_query=None, sort_by_key=(), page_current=0, page_size=5
)
assert page.height <= 5
assert total >= page.height
assert isinstance(total_unique, int)
def test_fetch_page_sql_applies_filter(flask_app):
from src.utils import table
with flask_app.app_context():
page, total, total_unique = table._fetch_page_sql(
filter_query="{uid} icontains __ne_matche_rien__",
sort_by_key=(),
page_current=0,
page_size=20,
)
assert total == 0
assert page.height == 0
def test_fetch_page_sql_post_processes_links(flask_app):
from src.utils import table
with flask_app.app_context():
page, _, _ = table._fetch_page_sql(
filter_query=None, sort_by_key=(), page_current=0, page_size=1
)
if page.height > 0:
assert "<a href" in page["uid"][0]
def test_postprocess_page_produces_same_result_as_full_then_slice(
flask_app, sample_lff
):