diff --git a/src/utils/table.py b/src/utils/table.py
index 2c0d2ea..9e47142 100644
--- a/src/utils/table.py
+++ b/src/utils/table.py
@@ -5,7 +5,7 @@ import polars as pl
from dash import no_update
from polars import selectors as cs
-from src.db import query_marches, schema
+from src.db import count_marches, count_unique_marches, query_marches, schema
from src.utils import logger
from src.utils.cache import cache
from src.utils.data import DATA_SCHEMA
@@ -376,28 +376,6 @@ def get_default_hidden_columns(page):
return hidden_columns
-@cache.memoize()
-def _load_filter_sort_postprocess(filter_query, sort_by_key):
- logger.debug(
- f"Cache miss — recomputing for filter={filter_query!r} sort={sort_by_key!r}"
- )
-
- lff: pl.LazyFrame = query_marches().lazy()
-
- if filter_query:
- lff = filter_table_data(lff, filter_query)
-
- if sort_by_key:
- sort_by = [
- {"column_id": col, "direction": direction} for col, direction in sort_by_key
- ]
- lff = sort_table_data(lff, sort_by)
-
- dff = table_postprocess(lff)
-
- return dff
-
-
def table_postprocess(lff) -> pl.DataFrame:
lff = lff.cast(pl.String)
lff = lff.fill_null("")
@@ -426,6 +404,48 @@ def postprocess_page(dff: pl.DataFrame) -> pl.DataFrame:
return dff
+@cache.memoize()
+def _fetch_page_sql(
+ filter_query: str | None,
+ sort_by_key: tuple,
+ page_current: int,
+ page_size: int,
+) -> tuple[pl.DataFrame, int, int]:
+ """Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule.
+
+ Retourne (page_dataframe_post_traitée, total_count, total_unique_count).
+ """
+ # Import local pour éviter une dépendance circulaire
+ # (src.utils.table_sql importe split_filter_part depuis src.utils.table).
+ from src.utils.table_sql import filter_query_to_sql, sort_by_to_sql
+
+ logger.debug(
+ f"Cache miss SQL — filter={filter_query!r} sort={sort_by_key!r} "
+ f"page={page_current} size={page_size}"
+ )
+
+ where_sql, params = filter_query_to_sql(filter_query or "", schema)
+
+ sort_by_dash = [
+ {"column_id": col, "direction": direction} for col, direction in sort_by_key
+ ]
+ order_by = sort_by_to_sql(sort_by_dash, schema) or None
+
+ total = count_marches(where_sql, params)
+ total_unique = count_unique_marches(where_sql, params)
+
+ page = query_marches(
+ where_sql=where_sql,
+ params=params,
+ order_by=order_by,
+ limit=page_size,
+ offset=page_current * page_size,
+ )
+
+ page = postprocess_page(page)
+ return page, total, total_unique
+
+
def prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
):
@@ -450,10 +470,16 @@ def prepare_table_data(
if data is None:
sort_by_key = normalize_sort_by(sort_by)
- dff: pl.DataFrame = _load_filter_sort_postprocess(
- filter_query=filter_query, sort_by_key=sort_by_key
+ dff, total, total_unique = _fetch_page_sql(
+ filter_query=filter_query,
+ sort_by_key=sort_by_key,
+ page_current=page_current,
+ page_size=page_size,
)
+ height = total
+ already_paginated = True
else:
+ already_paginated = False
if isinstance(data, list):
lff: pl.LazyFrame = pl.LazyFrame(
data, strict=False, infer_schema_length=5000
@@ -470,19 +496,19 @@ def prepare_table_data(
lff = sort_table_data(lff, sort_by)
dff: pl.DataFrame = table_postprocess(lff)
-
- height = dff.height
+ height = dff.height
+ total_unique = dff.select("uid").unique().height if "uid" in dff.columns else 0
if height > 0:
nb_rows = (
- f"{format_number(height)} lignes "
- f"({format_number(dff.select('uid').unique().height)} marchés)"
+ f"{format_number(height)} lignes ({format_number(total_unique)} marchés)"
)
else:
nb_rows = "0 lignes (0 marchés)"
- start_row = page_current * page_size
- dff = dff.slice(start_row, page_size)
+ if not already_paginated:
+ start_row = page_current * page_size
+ dff = dff.slice(start_row, page_size)
table_columns, tooltip = setup_table_columns(dff)
diff --git a/tests/test_table.py b/tests/test_table.py
index c47f967..65a9f52 100644
--- a/tests/test_table.py
+++ b/tests/test_table.py
@@ -83,7 +83,7 @@ def flask_app():
"""Minimal Flask app with SimpleCache so @cache.memoize() works in tests."""
from flask import Flask
- from utils.cache import cache
+ from src.utils.cache import cache
app = Flask(__name__)
cache.init_app(app, config={"CACHE_TYPE": "SimpleCache"})
@@ -95,7 +95,7 @@ def reset_cache(flask_app):
"""Ensure the flask-caching backend is empty between tests so that
cache-hit assertions are meaningful. Falls back to no-op when no
Flask app context is active (NullCache)."""
- from utils.cache import cache
+ from src.utils.cache import cache
with flask_app.app_context():
try:
@@ -106,60 +106,9 @@ def reset_cache(flask_app):
yield
-def test_load_filter_sort_postprocess_returns_dataframe(
- flask_app, monkeypatch, sample_lff
-):
+def test_prepare_table_data_returns_expected_tuple(flask_app):
from src.utils import table
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
- with flask_app.app_context():
- df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=())
-
- assert isinstance(df, pl.DataFrame)
- assert df.height == 1
- # All values must be strings after post-processing
- for col in df.columns:
- assert df.schema[col] == pl.String
-
-
-def test_load_filter_sort_postprocess_applies_filter(
- flask_app, monkeypatch, sample_lff
-):
- from src.utils import table
-
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
- with flask_app.app_context():
- df = table._load_filter_sort_postprocess(
- filter_query="{objet} icontains travaux", sort_by_key=()
- )
- assert df.height == 1
-
- df_empty = table._load_filter_sort_postprocess(
- filter_query="{objet} icontains nonexistent", sort_by_key=()
- )
- assert df_empty.height == 0
-
-
-def test_load_filter_sort_postprocess_adds_links(flask_app, monkeypatch, sample_lff):
- from src.utils import table
-
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
- with flask_app.app_context():
- df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=())
- # add_links injects an wrapper around uid, acheteur_nom, titulaire_nom
- assert "= 1
-def test_prepare_table_data_cleanup_trigger_for_non_tableau(
- monkeypatch, flask_app, sample_lff
-):
+def test_prepare_table_data_cleanup_trigger_for_non_tableau(flask_app):
"""Non-tableau pages still get a fresh uuid trigger, not no_update."""
from dash import no_update
from src.utils import table
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
with flask_app.app_context():
result = table.prepare_table_data(
data=None,
@@ -289,11 +234,11 @@ def test_prepare_table_data_with_external_data_does_not_use_cache(
sentinel["called"] = True
raise AssertionError("Memoized helper must not be called when data is provided")
- monkeypatch.setattr(table, "_load_filter_sort_postprocess", should_not_be_called)
+ monkeypatch.setattr(table, "_fetch_page_sql", should_not_be_called)
with flask_app.app_context():
table.prepare_table_data(
- data=sample_lff, # external LazyFrame
+ data=sample_lff,
data_timestamp=0,
filter_query=None,
page_current=0,
@@ -305,6 +250,44 @@ def test_prepare_table_data_with_external_data_does_not_use_cache(
assert sentinel["called"] is False
+def test_fetch_page_sql_respects_pagination(flask_app):
+ """New path: returns (page_dff, total_count, total_unique) via DuckDB."""
+ from src.utils import table
+
+ with flask_app.app_context():
+ page, total, total_unique = table._fetch_page_sql(
+ filter_query=None, sort_by_key=(), page_current=0, page_size=5
+ )
+ assert page.height <= 5
+ assert total >= page.height
+ assert isinstance(total_unique, int)
+
+
+def test_fetch_page_sql_applies_filter(flask_app):
+ from src.utils import table
+
+ with flask_app.app_context():
+ page, total, total_unique = table._fetch_page_sql(
+ filter_query="{uid} icontains __ne_matche_rien__",
+ sort_by_key=(),
+ page_current=0,
+ page_size=20,
+ )
+ assert total == 0
+ assert page.height == 0
+
+
+def test_fetch_page_sql_post_processes_links(flask_app):
+ from src.utils import table
+
+ with flask_app.app_context():
+ page, _, _ = table._fetch_page_sql(
+ filter_query=None, sort_by_key=(), page_current=0, page_size=1
+ )
+ if page.height > 0:
+ assert "