diff --git a/CLAUDE.md b/CLAUDE.md
index 509783b..5af8a09 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -10,27 +10,38 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
### Setup
+Setting up the virtual environment:
+
```bash
-cp template.env .env # then customize .env
+python -m venv .venv # s'il n'existe pas déjà
+source .venv/bin/activate
+rtk pip install -U pip > /dev/null 2>&1
+rtk pip install -e . --group=dev
+```
+
+Environment variables:
+
+```bash
+cp .template.env .env # then customize .env
```
### Development
```bash
-uv run run.py # starts Dash with debug=True and hot reload
+python run.py # starts Dash app
```
### Production
```bash
-uv run gunicorn app:server
+gunicorn app:server
```
### Tests
```bash
-rtk uv run pytest # run all tests (Selenium-based integration tests)
-rtk uv run pytest tests/test_main.py::test_001_logo_and_search # run a single test
+rtk pytest # run all tests (some are Selenium-based integration tests)
+rtk pytest tests/test_main.py::test_001_logo_and_search # run a single test
```
Tests require a running Chrome/Chromium browser. They use `DashComposite` from `dash[testing]` with Selenium WebDriver.
@@ -40,12 +51,12 @@ Tests require a running Chrome/Chromium browser. They use `DashComposite` from `
### Multi-page Dash app
- `src/app.py` — creates the Dash app instance, navbar, SEO endpoints (robots.txt, sitemap.xml), Matomo analytics
-- `src/pages/*.py` — each page registers itself with `@register_page()` and owns its own layout and callbacks
+- `src/pages/*.py` — each page registers itself with `@register_page()` and o.wns its own layout and callbacks
- `run.py` — dev entry point; exports `server` (Flask) for gunicorn
### Module imports
-- always import modules from the app starting with `src.` (e.g. `src.utils.`, `src.pages.recherche`, etc.)
+- always import modules from the app starting with `src.` (e.g. `src.utils.`, `src.pages.recherche`, etc.), NOT `utils.cache` or `pages.observatoire`.
### Key pages
@@ -60,9 +71,9 @@ Tests require a running Chrome/Chromium browser. They use `DashComposite` from `
### Data layer
-- Data is stored as **Parquet** and loaded with **Polars** (fast columnar operations)
+- Data is stored as **Parquet** at rest, possibly in DuckDB, loaded in DuckDB, served from DuckDB for big queries and manipulated with **Polars** for the remaining steps
- Path set via `DATA_FILE_PARQUET_PATH` env var; tests use `tests/test.parquet`
-- `src/utils.py` — filtering helpers, search (`search_org`), link generation, geographic data loading
+- `src/util/*.py` — helpers shared by other modules, search (`search_org`), link generation, geographic data loading
- `src/callbacks.py` — shared Dash callbacks (e.g. `get_top_org_table`)
- `src/figures.py` — chart and map components (Plotly Express, Dash Leaflet with marker clustering)
- a Parquet file with production data is located at `../decp-processing/decp_prod.parquet` (~ 1,5 million records)
diff --git a/src/db.py b/src/db.py
index 9d96872..9290e48 100644
--- a/src/db.py
+++ b/src/db.py
@@ -135,10 +135,11 @@ def get_cursor() -> duckdb.DuckDBPyConnection:
def query_marches(
where_sql: str = "TRUE",
- params: tuple = (),
+ params: tuple | list = (),
columns: list[str] | None = None,
order_by: str | None = None,
limit: int | None = None,
+ offset: int | None = None,
) -> pl.DataFrame:
"""Run a parameterized SELECT against the decp table and return Polars.
@@ -152,4 +153,20 @@ def query_marches(
sql += f" ORDER BY {order_by}"
if limit is not None:
sql += f" LIMIT {int(limit)}"
+ if offset is not None:
+ sql += f" OFFSET {int(offset)}"
return get_cursor().execute(sql, list(params)).pl()
+
+
+def count_marches(where_sql: str = "TRUE", params: tuple | list = ()) -> int:
+ """Retourne le nombre de lignes correspondant à where_sql."""
+ sql = f"SELECT COUNT(*) FROM decp WHERE {where_sql}"
+ result = get_cursor().execute(sql, list(params)).fetchone()
+ return int(result[0]) if result else 0
+
+
+def count_unique_marches(where_sql: str = "TRUE", params: tuple | list = ()) -> int:
+ """Retourne le nombre de uid distincts correspondant à where_sql."""
+ sql = f"SELECT COUNT(DISTINCT uid) FROM decp WHERE {where_sql}"
+ result = get_cursor().execute(sql, list(params)).fetchone()
+ return int(result[0]) if result else 0
diff --git a/src/utils/table.py b/src/utils/table.py
index 096acf5..e3b59a8 100644
--- a/src/utils/table.py
+++ b/src/utils/table.py
@@ -5,7 +5,7 @@ import polars as pl
from dash import no_update
from polars import selectors as cs
-from src.db import query_marches, schema
+from src.db import count_marches, count_unique_marches, query_marches, schema
from src.utils import logger
from src.utils.cache import cache
from src.utils.data import DATA_SCHEMA
@@ -376,28 +376,6 @@ def get_default_hidden_columns(page):
return hidden_columns
-@cache.memoize()
-def _load_filter_sort_postprocess(filter_query, sort_by_key):
- logger.debug(
- f"Cache miss — recomputing for filter={filter_query!r} sort={sort_by_key!r}"
- )
-
- lff: pl.LazyFrame = query_marches().lazy()
-
- if filter_query:
- lff = filter_table_data(lff, filter_query)
-
- if sort_by_key:
- sort_by = [
- {"column_id": col, "direction": direction} for col, direction in sort_by_key
- ]
- lff = sort_table_data(lff, sort_by)
-
- dff = table_postprocess(lff)
-
- return dff
-
-
def table_postprocess(lff) -> pl.DataFrame:
lff = lff.cast(pl.String)
lff = lff.fill_null("")
@@ -410,6 +388,64 @@ def table_postprocess(lff) -> pl.DataFrame:
return dff
+def postprocess_page(dff: pl.DataFrame) -> pl.DataFrame:
+ """Post-traitement à appliquer sur une page déjà paginée.
+
+ Équivalent à table_postprocess mais prend une DataFrame (pas un LazyFrame)
+ et ne matérialise rien de plus. À appeler après que la pagination ait été
+ poussée en SQL.
+ """
+ dff = dff.with_columns(pl.all().cast(pl.String).fill_null(""))
+ dff = add_links(dff)
+ if "sourceFile" in dff.columns:
+ dff = add_resource_link(dff)
+ if dff.height > 0:
+ dff = format_values(dff)
+ return dff
+
+
+@cache.memoize()
+def _fetch_page_sql(
+ filter_query: str | None,
+ sort_by_key: tuple,
+ page_current: int,
+ page_size: int,
+) -> tuple[pl.DataFrame, int, int]:
+ """Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule.
+
+ Retourne (page_dataframe_post_traitée, total_count, total_unique_count).
+ """
+ # Import local pour éviter une dépendance circulaire
+ # (src.utils.table_sql importe split_filter_part depuis src.utils.table).
+ from src.utils.table_sql import filter_query_to_sql, sort_by_to_sql
+
+ logger.debug(
+ f"Cache miss SQL — filter={filter_query!r} sort={sort_by_key!r} "
+ f"page={page_current} size={page_size}"
+ )
+
+ where_sql, params = filter_query_to_sql(filter_query or "", schema)
+
+ sort_by_dash = [
+ {"column_id": col, "direction": direction} for col, direction in sort_by_key
+ ]
+ order_by = sort_by_to_sql(sort_by_dash, schema) or None
+
+ total = count_marches(where_sql, params)
+ total_unique = count_unique_marches(where_sql, params)
+
+ page = query_marches(
+ where_sql=where_sql,
+ params=params,
+ order_by=order_by,
+ limit=page_size,
+ offset=page_current * page_size,
+ )
+
+ page = postprocess_page(page)
+ return page, total, total_unique
+
+
def prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
):
@@ -434,10 +470,15 @@ def prepare_table_data(
if data is None:
sort_by_key = normalize_sort_by(sort_by)
- dff: pl.DataFrame = _load_filter_sort_postprocess(
- filter_query=filter_query, sort_by_key=sort_by_key
+ dff, height, total_unique = _fetch_page_sql(
+ filter_query=filter_query,
+ sort_by_key=sort_by_key,
+ page_current=page_current,
+ page_size=page_size,
)
+ already_paginated = True
else:
+ already_paginated = False
if isinstance(data, list):
lff: pl.LazyFrame = pl.LazyFrame(
data, strict=False, infer_schema_length=5000
@@ -454,19 +495,19 @@ def prepare_table_data(
lff = sort_table_data(lff, sort_by)
dff: pl.DataFrame = table_postprocess(lff)
-
- height = dff.height
+ height = dff.height
+ total_unique = dff.select("uid").unique().height if "uid" in dff.columns else 0
if height > 0:
nb_rows = (
- f"{format_number(height)} lignes "
- f"({format_number(dff.select('uid').unique().height)} marchés)"
+ f"{format_number(height)} lignes ({format_number(total_unique)} marchés)"
)
else:
nb_rows = "0 lignes (0 marchés)"
- start_row = page_current * page_size
- dff = dff.slice(start_row, page_size)
+ if not already_paginated:
+ start_row = page_current * page_size
+ dff = dff.slice(start_row, page_size)
table_columns, tooltip = setup_table_columns(dff)
diff --git a/src/utils/table_sql.py b/src/utils/table_sql.py
new file mode 100644
index 0000000..67d7dcc
--- /dev/null
+++ b/src/utils/table_sql.py
@@ -0,0 +1,102 @@
+import polars as pl
+
+from src.utils import logger
+from src.utils.table import split_filter_part
+
+
+def filter_query_to_sql(filter_query: str, schema: pl.Schema) -> tuple[str, list]:
+ """Traduit le DSL de filtres de dash_table.DataTable en fragment SQL DuckDB.
+
+ Retourne (where_clause, params) où where_clause est un fragment à injecter
+ après WHERE et params est la liste des valeurs à passer à
+ cursor.execute(sql, params). Les identifiants de colonnes sont validés
+ contre le schéma fourni ; jamais concaténés avec des valeurs utilisateur.
+ """
+ if not filter_query:
+ return "TRUE", []
+
+ clauses: list[str] = []
+ params: list = []
+
+ for part in filter_query.split(" && "):
+ col_name, operator, raw_value = split_filter_part(part)
+ if not isinstance(col_name, str) or not isinstance(raw_value, str):
+ continue
+
+ if col_name not in schema.names():
+ logger.warning(f"Colonne inconnue ignorée : {col_name!r}")
+ continue
+
+ col_type = schema[col_name]
+ is_numeric = col_type.is_numeric()
+ col_is_date = col_type == pl.Date
+ quoted_col = f'"{col_name}"'
+
+ if is_numeric:
+ try:
+ value = int(raw_value) if col_type.is_integer() else float(raw_value)
+ except ValueError:
+ logger.warning(f"Valeur numérique invalide ignorée : {raw_value!r}")
+ continue
+
+ if operator == "contains":
+ clauses.append(f"{quoted_col} IS NOT NULL AND {quoted_col} = ?")
+ elif operator == ">":
+ clauses.append(f"{quoted_col} IS NOT NULL AND {quoted_col} > ?")
+ elif operator == "<":
+ clauses.append(f"{quoted_col} IS NOT NULL AND {quoted_col} < ?")
+ else:
+ logger.warning(f"Opérateur invalide pour numérique : {operator!r}")
+ continue
+ params.append(value)
+ continue
+
+ # String / Date : toujours traité comme texte (parité avec Polars)
+ value = raw_value.strip('"')
+
+ if operator == "contains":
+ if value.endswith("*") and not value.startswith("*"):
+ like = value[:-1] + "%"
+ elif value.startswith("*") and not value.endswith("*"):
+ like = "%" + value[1:]
+ else:
+ like = "%" + value + "%"
+ target = f"CAST({quoted_col} AS VARCHAR)" if col_is_date else quoted_col
+ clauses.append(
+ f"{quoted_col} IS NOT NULL AND {target} <> '' AND {target} ILIKE ?"
+ )
+ params.append(like)
+ elif operator in (">", "<"):
+ target = f"CAST({quoted_col} AS VARCHAR)" if col_is_date else quoted_col
+ clauses.append(f"{quoted_col} IS NOT NULL AND {target} {operator} ?")
+ params.append(value)
+ else:
+ logger.warning(f"Opérateur invalide pour chaîne : {operator!r}")
+ continue
+
+ if not clauses:
+ return "TRUE", []
+ return " AND ".join(clauses), params
+
+
+def sort_by_to_sql(sort_by: list[dict] | None, schema: pl.Schema) -> str:
+ """Traduit sort_by (format Dash) en clause ORDER BY DuckDB.
+
+ Retourne '' si pas de tri (aucun ORDER BY à ajouter).
+ """
+ if not sort_by:
+ return ""
+
+ fragments: list[str] = []
+ for entry in sort_by:
+ col = entry.get("column_id")
+ direction = entry.get("direction")
+ if col not in schema.names():
+ logger.warning(f"Tri sur colonne inconnue ignoré : {col!r}")
+ continue
+ if direction not in ("asc", "desc"):
+ logger.warning(f"Tri sur direction inconnue ignoré : {direction!r}")
+ continue
+ fragments.append(f'"{col}" {direction.upper()} NULLS LAST')
+
+ return ", ".join(fragments)
diff --git a/tests/test_db.py b/tests/test_db.py
index 2edd1c5..0ca9e36 100644
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -206,6 +206,38 @@ def test_query_marches_returns_polars_frame(built_db, monkeypatch):
assert set(frame["uid"].to_list()) == {"1", "2"}
+def test_count_marches_returns_total_without_filter():
+ from src.db import count_marches
+
+ n = count_marches()
+ assert isinstance(n, int)
+ assert n > 0
+
+
+def test_count_marches_with_filter():
+ from src.db import count_marches
+
+ n = count_marches('"uid" = ?', ["__nonexistent__"])
+ assert n == 0
+
+
+def test_count_unique_marches_respects_distinct():
+ from src.db import count_unique_marches
+
+ n = count_unique_marches()
+ assert isinstance(n, int)
+ assert n > 0
+
+
+def test_query_marches_with_offset():
+ from src.db import query_marches
+
+ page_0 = query_marches(limit=2, offset=0)
+ page_1 = query_marches(limit=2, offset=2)
+ if page_0.height == 2 and page_1.height >= 1:
+ assert set(page_0["uid"].to_list()).isdisjoint(set(page_1["uid"].to_list()))
+
+
def test_concurrent_build_serialized(tmp_path):
"""Multiple threads calling _ensure_database must serialize via flock.
diff --git a/tests/test_table.py b/tests/test_table.py
index cc89386..65a9f52 100644
--- a/tests/test_table.py
+++ b/tests/test_table.py
@@ -83,7 +83,7 @@ def flask_app():
"""Minimal Flask app with SimpleCache so @cache.memoize() works in tests."""
from flask import Flask
- from utils.cache import cache
+ from src.utils.cache import cache
app = Flask(__name__)
cache.init_app(app, config={"CACHE_TYPE": "SimpleCache"})
@@ -95,7 +95,7 @@ def reset_cache(flask_app):
"""Ensure the flask-caching backend is empty between tests so that
cache-hit assertions are meaningful. Falls back to no-op when no
Flask app context is active (NullCache)."""
- from utils.cache import cache
+ from src.utils.cache import cache
with flask_app.app_context():
try:
@@ -106,60 +106,9 @@ def reset_cache(flask_app):
yield
-def test_load_filter_sort_postprocess_returns_dataframe(
- flask_app, monkeypatch, sample_lff
-):
+def test_prepare_table_data_returns_expected_tuple(flask_app):
from src.utils import table
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
- with flask_app.app_context():
- df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=())
-
- assert isinstance(df, pl.DataFrame)
- assert df.height == 1
- # All values must be strings after post-processing
- for col in df.columns:
- assert df.schema[col] == pl.String
-
-
-def test_load_filter_sort_postprocess_applies_filter(
- flask_app, monkeypatch, sample_lff
-):
- from src.utils import table
-
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
- with flask_app.app_context():
- df = table._load_filter_sort_postprocess(
- filter_query="{objet} icontains travaux", sort_by_key=()
- )
- assert df.height == 1
-
- df_empty = table._load_filter_sort_postprocess(
- filter_query="{objet} icontains nonexistent", sort_by_key=()
- )
- assert df_empty.height == 0
-
-
-def test_load_filter_sort_postprocess_adds_links(flask_app, monkeypatch, sample_lff):
- from src.utils import table
-
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
- with flask_app.app_context():
- df = table._load_filter_sort_postprocess(filter_query=None, sort_by_key=())
- # add_links injects an wrapper around uid, acheteur_nom, titulaire_nom
- assert "= 1
-def test_prepare_table_data_cleanup_trigger_for_non_tableau(
- monkeypatch, flask_app, sample_lff
-):
+def test_prepare_table_data_cleanup_trigger_for_non_tableau(flask_app):
"""Non-tableau pages still get a fresh uuid trigger, not no_update."""
from dash import no_update
from src.utils import table
- monkeypatch.setattr(table, "query_marches", lambda: sample_lff.collect())
-
with flask_app.app_context():
result = table.prepare_table_data(
data=None,
@@ -289,11 +234,11 @@ def test_prepare_table_data_with_external_data_does_not_use_cache(
sentinel["called"] = True
raise AssertionError("Memoized helper must not be called when data is provided")
- monkeypatch.setattr(table, "_load_filter_sort_postprocess", should_not_be_called)
+ monkeypatch.setattr(table, "_fetch_page_sql", should_not_be_called)
with flask_app.app_context():
table.prepare_table_data(
- data=sample_lff, # external LazyFrame
+ data=sample_lff,
data_timestamp=0,
filter_query=None,
page_current=0,
@@ -303,3 +248,57 @@ def test_prepare_table_data_with_external_data_does_not_use_cache(
)
assert sentinel["called"] is False
+
+
+def test_fetch_page_sql_respects_pagination(flask_app):
+ """New path: returns (page_dff, total_count, total_unique) via DuckDB."""
+ from src.utils import table
+
+ with flask_app.app_context():
+ page, total, total_unique = table._fetch_page_sql(
+ filter_query=None, sort_by_key=(), page_current=0, page_size=5
+ )
+ assert page.height <= 5
+ assert total >= page.height
+ assert isinstance(total_unique, int)
+
+
+def test_fetch_page_sql_applies_filter(flask_app):
+ from src.utils import table
+
+ with flask_app.app_context():
+ page, total, total_unique = table._fetch_page_sql(
+ filter_query="{uid} icontains __ne_matche_rien__",
+ sort_by_key=(),
+ page_current=0,
+ page_size=20,
+ )
+ assert total == 0
+ assert page.height == 0
+
+
+def test_fetch_page_sql_post_processes_links(flask_app):
+ from src.utils import table
+
+ with flask_app.app_context():
+ page, _, _ = table._fetch_page_sql(
+ filter_query=None, sort_by_key=(), page_current=0, page_size=1
+ )
+ if page.height > 0:
+ assert " \'\' AND "objet" ILIKE ?'
+ assert params == ["%travaux%"]
+
+
+def test_icontains_with_trailing_wildcard_is_starts_with():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql(
+ "{acheteur_id} icontains 24350013900189*", SCHEMA
+ )
+ assert (
+ where
+ == '"acheteur_id" IS NOT NULL AND "acheteur_id" <> \'\' AND "acheteur_id" ILIKE ?'
+ )
+ assert params == ["24350013900189%"]
+
+
+def test_icontains_with_leading_wildcard_is_ends_with():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{uid} icontains *2024", SCHEMA)
+ assert where == '"uid" IS NOT NULL AND "uid" <> \'\' AND "uid" ILIKE ?'
+ assert params == ["%2024"]
+
+
+def test_numeric_greater_than():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{montant} i> 40000", SCHEMA)
+ assert where == '"montant" IS NOT NULL AND "montant" > ?'
+ assert params == [40000.0]
+
+
+def test_numeric_less_than():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{montant} i< 1000", SCHEMA)
+ assert where == '"montant" IS NOT NULL AND "montant" < ?'
+ assert params == [1000.0]
+
+
+def test_numeric_equality_via_icontains():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{dureeMois} icontains 12", SCHEMA)
+ assert where == '"dureeMois" IS NOT NULL AND "dureeMois" = ?'
+ assert params == [12]
+
+
+def test_date_column_treated_as_string_ilike():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{dateNotification} icontains 2024*", SCHEMA)
+ assert "ILIKE" in where
+ assert params == ["2024%"]
+
+
+def test_multiple_filters_joined_by_and():
+ from src.utils.table_sql import filter_query_to_sql
+
+ filter_query = "{objet} icontains voirie && {montant} i> 40000"
+ where, params = filter_query_to_sql(filter_query, SCHEMA)
+ assert " AND " in where
+ assert params == ["%voirie%", 40000.0]
+
+
+def test_invalid_numeric_value_is_skipped():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{montant} i> notanumber", SCHEMA)
+ assert where == "TRUE"
+ assert params == []
+
+
+def test_unknown_column_is_skipped():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql("{inexistant} icontains foo", SCHEMA)
+ assert where == "TRUE"
+ assert params == []
+
+
+def test_escapes_identifier_with_quotes_not_concatenation():
+ from src.utils.table_sql import filter_query_to_sql
+
+ where, params = filter_query_to_sql(
+ "{objet} icontains '; DROP TABLE decp; --", SCHEMA
+ )
+ assert "DROP TABLE" not in where
+ assert any("DROP TABLE" in str(p) for p in params)
+
+
+def test_sort_by_empty():
+ from src.utils.table_sql import sort_by_to_sql
+
+ assert sort_by_to_sql([], SCHEMA) == ""
+ assert sort_by_to_sql(None, SCHEMA) == ""
+
+
+def test_sort_by_single_column_desc():
+ from src.utils.table_sql import sort_by_to_sql
+
+ result = sort_by_to_sql([{"column_id": "montant", "direction": "desc"}], SCHEMA)
+ assert result == '"montant" DESC NULLS LAST'
+
+
+def test_sort_by_multiple_columns_preserves_order():
+ from src.utils.table_sql import sort_by_to_sql
+
+ result = sort_by_to_sql(
+ [
+ {"column_id": "dateNotification", "direction": "desc"},
+ {"column_id": "montant", "direction": "asc"},
+ ],
+ SCHEMA,
+ )
+ assert result == '"dateNotification" DESC NULLS LAST, "montant" ASC NULLS LAST'
+
+
+def test_sort_by_ignores_unknown_column():
+ from src.utils.table_sql import sort_by_to_sql
+
+ result = sort_by_to_sql([{"column_id": "fake", "direction": "asc"}], SCHEMA)
+ assert result == ""
diff --git a/uv.lock b/uv.lock
index 2c14012..0fd8ca1 100644
--- a/uv.lock
+++ b/uv.lock
@@ -760,7 +760,7 @@ wheels = [
[[package]]
name = "decp-info"
-version = "2.7.2"
+version = "2.7.3"
source = { virtual = "." }
dependencies = [
{ name = "dash", extra = ["compress"] },