colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

perf(tableau): memoize filter+sort+postprocess pipeline (#72)

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Colin Maudry
2026-04-19 22:50:32 +02:00
parent 0abbd982ea
commit b0d2aca4ff
2 changed files with 188 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/table.py
+38 -44
View File
@@ -10,7 +10,7 @@ from src.db import query_marches, schema
from src.utils import logger
from src.utils.data import DATA_SCHEMA
from src.utils.frontend import get_button_properties
from src.utils.tracking import track_search # noqa: F401
from src.utils.tracking import track_search
def split_filter_part(filter_part):
@@ -420,66 +420,60 @@ def prepare_table_data(
:param source_table:
:return:
"""
logger.debug(" + + + + + + + + + + + + + + + + + + ")
if os.getenv("DEVELOPMENT").lower() == "true":
logger.debug(" + + + + + + + + + + + + + + + + + + ")
trigger_cleanup = no_update
# Récupération des données
if isinstance(data, list):
lff: pl.LazyFrame = pl.LazyFrame(data, strict=False, infer_schema_length=5000)
elif isinstance(data, pl.LazyFrame):
lff = data
else:
lff: pl.LazyFrame = query_marches().lazy()
# Application des filtres
if filter_query:
lff = filter_table_data(lff, filter_query, source_table)
trigger_cleanup = no_update if source_table == "tableau" else str(uuid.uuid4())
track_search(filter_query, source_table)
# Application des tris
if sort_by and len(sort_by) > 0:
lff = sort_table_data(lff, sort_by)
trigger_cleanup = no_update if source_table == "tableau" else str(uuid.uuid4())
if data is None:
sort_by_key = normalize_sort_by(sort_by)
dff: pl.DataFrame = _load_filter_sort_postprocess(
filter_query=filter_query, sort_by_key=sort_by_key
)
else:
if isinstance(data, list):
lff: pl.LazyFrame = pl.LazyFrame(
data, strict=False, infer_schema_length=5000
)
elif isinstance(data, pl.LazyFrame):
lff = data
else:
lff = query_marches().lazy()
if filter_query:
lff = filter_table_data(lff, filter_query, source_table)
if sort_by and len(sort_by) > 0:
lff = sort_table_data(lff, sort_by)
dff = lff.collect()
dff = dff.cast(pl.String)
dff = dff.fill_null("")
dff = add_links(dff)
if "sourceFile" in dff.columns:
dff = add_resource_link(dff)
if dff.height > 0:
dff = format_values(dff)
# Matérialisation des filtres
dff: pl.DataFrame = lff.collect()
height = dff.height
if height > 0:
nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)"
nb_rows = (
f"{format_number(height)} lignes "
f"({format_number(dff.select('uid').unique().height)} marchés)"
)
else:
nb_rows = "0 lignes (0 marchés)"
# Pagination des données
start_row = page_current * page_size
# end_row = (page_current + 1) * page_size
dff = dff.slice(start_row, page_size)
# Tout devient string
dff = dff.cast(pl.String)
# Remplace les strings null par "", mais pas les numeric null
dff = dff.fill_null("")
# Ajout des liens vers les pages de détails
dff = add_links(dff)
# Ajout des liens vers les fichiers Open Data
if "sourceFile" in dff.columns:
dff = add_resource_link(dff)
# Formatage des montants
if height > 0:
dff = format_values(dff)
# Récupération des colonnes et tooltip
table_columns, tooltip = setup_table_columns(dff)
dicts = dff.to_dicts()
# Propriétés du bouton de téléchargement
download_disabled, download_text, download_title = get_button_properties(height)
return (
tests/test_table.py
+150
View File
@@ -155,3 +155,153 @@ def test_load_filter_sort_postprocess_adds_links(flask_app, monkeypatch, sample_
assert "<a href" in df["uid"][0]
assert "<a href" in df["acheteur_nom"][0]
assert "<a href" in df["titulaire_nom"][0]
def test_prepare_table_data_returns_expected_tuple(monkeypatch, flask_app, sample_lff):
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
result = table.prepare_table_data(
data=None,
data_timestamp=5,
filter_query=None,
page_current=0,
page_size=20,
sort_by=[],
source_table="tableau",
)
# Same arity as before: 9 outputs
assert len(result) == 9
dicts, columns, tooltip, ts, nb_rows, dl_disabled, dl_text, dl_title, cleanup = (
result
)
assert isinstance(dicts, list)
assert ts == 6 # data_timestamp + 1 must still increment
assert "1 lignes" in nb_rows
def test_prepare_table_data_calls_track_search_on_filter(
monkeypatch, flask_app, sample_lff
):
from src.utils import table
calls = []
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
monkeypatch.setattr(table, "track_search", lambda *a, **kw: calls.append(a))
with flask_app.app_context():
table.prepare_table_data(
data=None,
data_timestamp=0,
filter_query="{objet} icontains travaux",
page_current=0,
page_size=20,
sort_by=[],
source_table="tableau",
)
assert calls == [("{objet} icontains travaux", "tableau")]
def test_prepare_table_data_paginates_without_recomputing(
monkeypatch, flask_app, sample_lff
):
"""Two calls with same filter+sort but different pages must invoke
the inner heavy work only once."""
from src.utils import table
call_count = {"n": 0}
real_query = sample_lff.collect()
def counting_query():
call_count["n"] += 1
return real_query
monkeypatch.setattr(table, "query_marches", counting_query)
with flask_app.app_context():
# First call: cache miss
table.prepare_table_data(
data=None,
data_timestamp=0,
filter_query=None,
page_current=0,
page_size=10,
sort_by=[],
source_table="tableau",
)
first_count = call_count["n"]
# Second call, different page: cache hit, query_marches must NOT fire again
table.prepare_table_data(
data=None,
data_timestamp=0,
filter_query=None,
page_current=1,
page_size=10,
sort_by=[],
source_table="tableau",
)
assert call_count["n"] == first_count, (
"query_marches was called again — pagination triggered cache miss"
)
def test_prepare_table_data_cleanup_trigger_for_non_tableau(
monkeypatch, flask_app, sample_lff
):
"""Non-tableau pages still get a fresh uuid trigger, not no_update."""
from dash import no_update
from src.utils import table
monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
with flask_app.app_context():
result = table.prepare_table_data(
data=None,
data_timestamp=0,
filter_query="{objet} icontains travaux",
page_current=0,
page_size=20,
sort_by=[],
source_table="acheteur",
)
cleanup = result[8]
assert cleanup is not no_update
assert isinstance(cleanup, str)
assert len(cleanup) >= 32 # uuid4 hex string
def test_prepare_table_data_with_external_data_does_not_use_cache(
monkeypatch, flask_app, sample_lff
):
"""When a caller passes data (acheteur/titulaire/observatoire path),
bypass the memoized helper entirely."""
from src.utils import table
sentinel = {"called": False}
def should_not_be_called(*a, **kw):
sentinel["called"] = True
raise AssertionError("Memoized helper must not be called when data is provided")
monkeypatch.setattr(table, "_load_filter_sort_postprocess", should_not_be_called)
with flask_app.app_context():
table.prepare_table_data(
data=sample_lff, # external LazyFrame
data_timestamp=0,
filter_query=None,
page_current=0,
page_size=20,
sort_by=[],
source_table="acheteur",
)
assert sentinel["called"] is False