Merge branch 'feat/duckdb-migration' into dev

.gitignore

@@ -6,3 +6,8 @@ __pycache__
 .worktrees
 build
 .env
+
+# DuckDB runtime artifacts (regenerated from decp_prod.parquet at startup)
+**/decp.duckdb
+**/decp.duckdb.tmp
+**/decp.duckdb.lock

docs/superpowers/specs/2026-03-21-observatoire-full-url-sharing-design.md

@@ -14,27 +14,28 @@ Flat query parameters with short, readable keys. Multi-value filters use repeate
 
 ## URL Parameter Mapping
 
-| Component ID | URL key | Type | Default (omitted) |
-|---|---|---|---|
-| `dashboard_year` | `annee` | single | `None` |
-| `dashboard_acheteur_id` | `acheteur_id` | single | `None` |
-| `dashboard_acheteur_categorie` | `acheteur_cat` | single | `None` |
-| `dashboard_acheteur_departement_code` | `acheteur_dept` | multi | `[]`/`None` |
-| `dashboard_titulaire_id` | `titulaire_id` | single | `None` |
-| `dashboard_titulaire_categorie` | `titulaire_cat` | single | `None` |
-| `dashboard_titulaire_departement_code` | `titulaire_dept` | multi | `[]`/`None` |
-| `dashboard_marche_type` | `type` | single | `None` |
-| `dashboard_marche_objet` | `objet` | single | `None` |
-| `dashboard_marche_code_cpv` | `cpv` | single | `None` |
-| `dashboard_montant_min` | `montant_min` | single (number) | `None` |
-| `dashboard_montant_max` | `montant_max` | single (number) | `None` |
-| `dashboard_marche_techniques` | `techniques` | multi | `[]`/`None` |
-| `dashboard_marche_innovant` | `innovant` | single | `"all"` |
-| `dashboard_marche_sousTraitanceDeclaree` | `sous_traitance` | single | `"all"` |
-| `dashboard_marche_considerationsSociales` | `social` | multi | `[]`/`None` |
-| `dashboard_marche_considerationsEnvironnementales` | `env` | multi | `[]`/`None` |
+| Component ID                                       | URL key          | Type            | Default (omitted) |
+| -------------------------------------------------- | ---------------- | --------------- | ----------------- |
+| `dashboard_year`                                   | `annee`          | single          | `None`            |
+| `dashboard_acheteur_id`                            | `acheteur_id`    | single          | `None`            |
+| `dashboard_acheteur_categorie`                     | `acheteur_cat`   | single          | `None`            |
+| `dashboard_acheteur_departement_code`              | `acheteur_dept`  | multi           | `[]`/`None`       |
+| `dashboard_titulaire_id`                           | `titulaire_id`   | single          | `None`            |
+| `dashboard_titulaire_categorie`                    | `titulaire_cat`  | single          | `None`            |
+| `dashboard_titulaire_departement_code`             | `titulaire_dept` | multi           | `[]`/`None`       |
+| `dashboard_marche_type`                            | `type`           | single          | `None`            |
+| `dashboard_marche_objet`                           | `objet`          | single          | `None`            |
+| `dashboard_marche_code_cpv`                        | `cpv`            | single          | `None`            |
+| `dashboard_montant_min`                            | `montant_min`    | single (number) | `None`            |
+| `dashboard_montant_max`                            | `montant_max`    | single (number) | `None`            |
+| `dashboard_marche_techniques`                      | `techniques`     | multi           | `[]`/`None`       |
+| `dashboard_marche_innovant`                        | `innovant`       | single          | `"all"`           |
+| `dashboard_marche_sousTraitanceDeclaree`           | `sous_traitance` | single          | `"all"`           |
+| `dashboard_marche_considerationsSociales`          | `social`         | multi           | `[]`/`None`       |
+| `dashboard_marche_considerationsEnvironnementales` | `env`            | multi           | `[]`/`None`       |
 
 Example URL:
+
 ```
 /observatoire?annee=2024&acheteur_id=12345678901234&acheteur_dept=75&acheteur_dept=13&montant_min=10000&innovant=oui
 ```
@@ -79,6 +80,7 @@ FILTER_PARAMS = [
 **Current:** Extracts only `acheteur_id` and `titulaire_id` from URL.
 
 **New:**
+
 - Iterates over `FILTER_PARAMS` to extract all values from `parse_qs`
 - For multi-value params: reads the full list from `parse_qs` (returns lists natively)
 - For number params (`montant_min`, `montant_max`): casts to `float`
@@ -102,12 +104,14 @@ Links generated by `add_links()` in `src/utils.py` (used on search results to li
 ### Fix broken test `test_010_observatoire_montant_filter`
 
 This test imports `_apply_filters` from `pages.observatoire`, which no longer exists (replaced by `prepare_dashboard_data` in `src/utils.py`). Fix:
+
 - Replace import with `from src.utils import prepare_dashboard_data`
 - Update the call to match `prepare_dashboard_data`'s signature: rename `marche_type` keyword to `type`, and add missing params `objet`, `code_cpv`, `techniques`, `marche_innovant`, `sous_traitance_declaree` (all as `None`)
 
 ### New test: multi-param URL round-trip
 
 Add a test that navigates to `/observatoire?annee=2024&acheteur_id=<test_id>&montant_min=10000` and verifies that:
+
 - `dashboard_year` dropdown shows "2024"
 - `dashboard_acheteur_id` input contains the test ID
 - `dashboard_montant_min` input contains "10000"

pyproject.toml

@@ -14,11 +14,13 @@ dependencies = [
   "xlsxwriter",
   "plotly[express]",
   "httpx",
-  "pandas",                    # utilisé pour la création de certains graphiques
+  "pandas", # utilisé pour la création de certains graphiques
   "unidecode",
   "dash-leaflet",
   "dash-extensions",
+  "duckdb",
   "flask-caching",
+  "pyarrow>=23.0.1",
 ]
 
 [project.optional-dependencies]

src/db.py

@@ -0,0 +1,161 @@
+import fcntl
+import logging
+import os
+from pathlib import Path
+from time import sleep
+
+import duckdb
+import polars as pl
+import polars.selectors as cs
+from polars.exceptions import ComputeError
+
+logger = logging.getLogger("decp.info")
+
+
+def should_rebuild(db_path: Path, parquet_path: Path) -> bool:
+    db_path = Path(db_path)
+    parquet_path = Path(parquet_path)
+    if not db_path.exists():
+        return True
+    dev = os.getenv("DEVELOPMENT", "False").lower() == "true"
+    force = os.getenv("REBUILD_DUCKDB", "False").lower() == "true"
+    if dev and not force:
+        return False
+    return parquet_path.stat().st_mtime > db_path.stat().st_mtime
+
+
+def _load_source_frame(parquet_path: Path) -> pl.DataFrame:
+    """Read the source parquet and apply the row-level transforms.
+
+    Kept here (not in utils.py) so src.db has no dependency on utils.
+    Mirrors the behavior previously in utils.get_decp_data().
+    """
+    try:
+        lff: pl.LazyFrame = pl.scan_parquet(str(parquet_path))
+    except ComputeError:
+        logger.info("Lecture du parquet échouée, nouvelle tentative dans 10s...")
+        sleep(10)
+        lff = pl.scan_parquet(str(parquet_path))
+
+    lff = lff.sort(by=["dateNotification", "uid"], descending=True, nulls_last=True)
+    lff = lff.filter(pl.col("donneesActuelles")).drop("donneesActuelles")
+
+    # booleans_to_strings: true → "oui", false → "non"
+    lff = lff.with_columns(
+        pl.col(cs.Boolean)
+        .cast(pl.String)
+        .str.replace("true", "oui")
+        .str.replace("false", "non")
+    )
+
+    for col in ["acheteur_nom", "titulaire_nom"]:
+        lff = lff.with_columns(
+            pl.when(pl.col(col).is_null())
+            .then(pl.lit("[Identifiant non reconnu dans la base INSEE]"))
+            .otherwise(pl.col(col))
+            .name.keep()
+        )
+
+    return lff.collect()
+
+
+def build_database(db_path: Path, parquet_path: Path) -> None:
+    """Build the DuckDB database atomically under an exclusive lock.
+
+    Caller MUST hold the fcntl.flock on the .lock file.
+    """
+    db_path = Path(db_path)
+    parquet_path = Path(parquet_path)
+    tmp_path = db_path.with_suffix(".duckdb.tmp")
+    staging_parquet = db_path.with_suffix(".staging.parquet")
+    if tmp_path.exists():
+        tmp_path.unlink()
+
+    logger.info(f"Construction de la base DuckDB à partir de {parquet_path}...")
+    frame = _load_source_frame(parquet_path)
+
+    # Write transformed frame as parquet so DuckDB can read it natively
+    # (avoids pyarrow dependency for the Polars→DuckDB handoff)
+    frame.write_parquet(str(staging_parquet))
+    try:
+        with duckdb.connect(str(tmp_path)) as w:
+            w.execute(
+                f"CREATE TABLE decp AS SELECT * FROM read_parquet('{staging_parquet}')"
+            )
+            w.execute(
+                "CREATE TABLE acheteurs_marches AS "
+                "SELECT DISTINCT uid, objet, acheteur_id FROM decp "
+                "ORDER BY acheteur_id"
+            )
+            w.execute(
+                "CREATE TABLE titulaires_marches AS "
+                "SELECT DISTINCT uid, objet, titulaire_id FROM decp "
+                "ORDER BY titulaire_id"
+            )
+            w.execute(
+                "CREATE TABLE acheteurs_departement AS "
+                "SELECT DISTINCT acheteur_id, acheteur_nom, acheteur_departement_code "
+                "FROM decp ORDER BY acheteur_nom"
+            )
+            w.execute(
+                "CREATE TABLE titulaires_departement AS "
+                "SELECT DISTINCT titulaire_id, titulaire_nom, titulaire_departement_code "
+                "FROM decp ORDER BY titulaire_nom"
+            )
+    finally:
+        if staging_parquet.exists():
+            staging_parquet.unlink()
+
+    os.replace(tmp_path, db_path)
+    logger.info(f"Base DuckDB construite : {db_path}")
+
+
+def _resolve_db_path() -> Path:
+    parquet = os.getenv("DATA_FILE_PARQUET_PATH")
+    if not parquet:
+        raise RuntimeError("DATA_FILE_PARQUET_PATH is not set")
+    return Path(parquet).parent / "decp.duckdb"
+
+
+def _ensure_database() -> Path:
+    db_path = _resolve_db_path()
+    parquet_path = Path(os.getenv("DATA_FILE_PARQUET_PATH"))
+    lock_path = db_path.with_suffix(".duckdb.lock")
+
+    with open(lock_path, "w") as lock_fd:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+        if should_rebuild(db_path, parquet_path):
+            build_database(db_path, parquet_path)
+    return db_path
+
+
+DB_PATH = _ensure_database()
+conn: duckdb.DuckDBPyConnection = duckdb.connect(str(DB_PATH), read_only=True)
+schema: pl.Schema = conn.execute("SELECT * FROM decp LIMIT 0").pl().schema
+
+
+def get_cursor() -> duckdb.DuckDBPyConnection:
+    """Return a per-request cursor that shares the process-wide connection."""
+    return conn.cursor()
+
+
+def query_marches(
+    where_sql: str = "TRUE",
+    params: tuple = (),
+    columns: list[str] | None = None,
+    order_by: str | None = None,
+    limit: int | None = None,
+) -> pl.DataFrame:
+    """Run a parameterized SELECT against the decp table and return Polars.
+
+    `where_sql` and `order_by` are trusted SQL fragments (callers are internal
+    code, never user input). `params` values are passed through DuckDB's
+    parameter binding.
+    """
+    cols = ", ".join(columns) if columns else "*"
+    sql = f"SELECT {cols} FROM decp WHERE {where_sql}"
+    if order_by:
+        sql += f" ORDER BY {order_by}"
+    if limit is not None:
+        sql += f" LIMIT {int(limit)}"
+    return get_cursor().execute(sql, list(params)).pl()

src/figures.py

@@ -12,11 +12,11 @@ import polars as pl
 from dash import dash_table, dcc, html
 from dash_extensions.javascript import Namespace
 
+from src.db import schema
 from src.utils import (
     add_links,
     data_schema,
     departements_geojson,
-    df,
     format_number,
     setup_table_columns,
 )
@@ -774,7 +774,7 @@ def make_column_picker(page: str):
             "name": data_schema[col]["title"],
             "description": data_schema[col]["description"],
         }
-        for col in df.columns
+        for col in schema.names()
     ]
     for column in table_columns:
         new_column = {

src/pages/acheteur.py

@@ -15,6 +15,7 @@ from dash import (
     register_page,
 )
 
+from src.db import query_marches, schema
 from src.figures import (
     DataTable,
     get_distance_histogram,
@@ -25,7 +26,6 @@ from src.figures import (
 )
 from src.utils import (
     columns,
-    df,
     df_acheteurs,
     filter_table_data,
     format_number,
@@ -70,7 +70,7 @@ datatable = html.Div(
         sort_action="custom",
         page_size=10,
         hidden_columns=[],
-        columns=[{"id": col, "name": col} for col in df.columns],
+        columns=[{"id": col, "name": col} for col in schema.names()],
     ),
 )
 
@@ -300,7 +300,7 @@ def update_acheteur_infos(url):
 def update_acheteur_stats(data):
     dff = pl.DataFrame(data, strict=False, infer_schema_length=5000)
     if dff.height == 0:
-        dff = pl.DataFrame(schema=df.collect_schema())
+        dff = pl.DataFrame(schema=schema)
     df_marches = dff.unique("id")
     nb_marches = format_number(df_marches.height)
     # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item()))
@@ -328,15 +328,13 @@ def update_acheteur_stats(data):
 )
 def get_acheteur_marches_data(url, ach_year: str) -> tuple:
     acheteur_siret = url.split("/")[-1]
-    lff = df.lazy()
-    lff = lff.filter(pl.col("acheteur_id") == acheteur_siret)
+    lff = query_marches("acheteur_id = ?", (acheteur_siret,)).lazy()
     if ach_year and ach_year != "Toutes les années":
-        ach_year: int = int(ach_year)
+        ach_year = int(ach_year)
         lff = lff.filter(pl.col("dateNotification").dt.year() == ach_year)
     lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True)
     dff: pl.DataFrame = lff.collect(engine="streaming")
     download_disabled, download_text, download_title = get_button_properties(dff.height)
-
     data = dff.to_dicts()
     return data, download_disabled, download_text, download_title
 

src/pages/arbre/departement.py

@@ -1,7 +1,7 @@
-import polars as pl
 from dash import Input, Output, callback, dcc, html, register_page
 
-from src.utils import departements, df_acheteurs_departement, df_titulaires_departement
+from src.db import get_cursor
+from src.utils import departements
 
 name = "Département"
 
@@ -39,29 +39,42 @@ def departement_marches(url):
     departement = url.split("/")[-1]
 
     def make_link_list(org_type) -> list:
-        link_list = []
-        if org_type == "acheteur":
-            df = df_acheteurs_departement
-        elif org_type == "titulaire":
-            df = df_titulaires_departement
-        else:
+        table = (
+            "acheteurs_departement"
+            if org_type == "acheteur"
+            else "titulaires_departement"
+            if org_type == "titulaire"
+            else None
+        )
+        if table is None:
             raise ValueError
+        col_prefix = org_type
+        rows = (
+            get_cursor()
+            .execute(
+                f"SELECT {col_prefix}_id, {col_prefix}_nom "
+                f"FROM {table} "
+                f"WHERE {col_prefix}_departement_code = ? "
+                f"ORDER BY {col_prefix}_nom",
+                [departement],
+            )
+            .fetchall()
+        )
 
-        df = df.filter(pl.col(f"{org_type}_departement_code") == departement)
-
-        for row in df.iter_rows(named=True):
+        link_list = []
+        for org_id, org_nom in rows:
             li = html.Li(
                 [
                     dcc.Link(
-                        row[f"{org_type}_nom"],
-                        href=url + f"/{org_type}/{row[f'{org_type}_id']}",
-                        title=f"Marchés publics de {row[f'{org_type}_nom']}",
+                        org_nom,
+                        href=url + f"/{org_type}/{org_id}",
+                        title=f"Marchés publics de {org_nom}",
                     ),
                     " ",
                     dcc.Link(
                         "(page dédiée)",
-                        href=f"/{org_type}s/{row[f'{org_type}_id']}",
-                        title=f"Page dédiée aux marchés publics de {row[f'{org_type}_nom']}",
+                        href=f"/{org_type}s/{org_id}",
+                        title=f"Page dédiée aux marchés publics de {org_nom}",
                     ),
                 ]
             )

src/pages/arbre/liste_marches_org.py

@@ -1,12 +1,8 @@
 import polars as pl
 from dash import Input, Output, callback, dcc, html, register_page
 
-from src.utils import (
-    df_acheteurs,
-    df_acheteurs_marches,
-    df_titulaires,
-    df_titulaires_marches,
-)
+from src.db import get_cursor
+from src.utils import df_acheteurs, df_titulaires
 
 name = "Liste des marchés publics"
 
@@ -68,28 +64,34 @@ def liste_marches(url):
     org_id = url.split("/")[-1]
 
     def make_link_list() -> list:
-        link_list = []
-        if org_type == "acheteur":
-            df = df_acheteurs_marches
-        elif org_type == "titulaire":
-            df = df_titulaires_marches
-        else:
+        table = (
+            "acheteurs_marches"
+            if org_type == "acheteur"
+            else "titulaires_marches"
+            if org_type == "titulaire"
+            else None
+        )
+        if table is None:
             raise ValueError
-
-        df = df.filter(pl.col(f"{org_type}_id") == org_id)
-
-        for row in df.iter_rows(named=True):
-            li = html.Li(
-                [
-                    dcc.Link(
-                        row["objet"],
-                        href=f"/marches/{row['uid']}",
-                        title=f"Marchés public attribué : {row['objet']}",
-                    )
-                ]
+        rows = (
+            get_cursor()
+            .execute(
+                f"SELECT uid, objet FROM {table} WHERE {org_type}_id = ?",
+                [org_id],
             )
-            link_list.append(li)
-        return link_list
+            .fetchall()
+        )
+
+        return [
+            html.Li(
+                dcc.Link(
+                    objet,
+                    href=f"/marches/{uid}",
+                    title=f"Marchés public attribué : {objet}",
+                )
+            )
+            for uid, objet in rows
+        ]
 
     nom, verbe = make_org_nom_verbe(org_type, org_id)
 

src/pages/marche.py

@@ -2,13 +2,12 @@ import json
 from datetime import datetime
 
 import dash_bootstrap_components as dbc
-import polars as pl
 from dash import Input, Output, callback, dcc, html, register_page
 from polars import selectors as cs
 
+from src.db import query_marches
 from src.utils import (
     data_schema,
-    df,
     format_values,
     make_org_jsonld,
     meta_content,
@@ -88,19 +87,17 @@ layout = [
 def get_marche_data(url) -> tuple[dict, list]:
     marche_uid = url.split("/")[-1]
 
-    # Récupération des données du marché à partir du df global
+    # Filtre SQL côté DuckDB, puis Polars pour le post-traitement
+    dff_marche = query_marches("uid = ?", (marche_uid,))
+    if dff_marche.height == 0:
+        return {}, []
 
-    lff = df.lazy()
-    lff = lff.filter(pl.col("uid") == pl.lit(marche_uid))
-
-    # Données des titulaires du marché
+    lff = dff_marche.lazy()
     dff_titulaires = lff.select(cs.starts_with("titulaire")).collect(engine="streaming")
+    dff_marche_unique = lff.unique("uid").collect(engine="streaming")
+    dff_marche_unique = format_values(dff_marche_unique)
 
-    # Données du marché
-    dff_marche = lff.unique("uid").collect(engine="streaming")
-    dff_marche = format_values(dff_marche)
-
-    return dff_marche.to_dicts()[0], dff_titulaires.to_dicts()
+    return dff_marche_unique.to_dicts()[0], dff_titulaires.to_dicts()
 
 
 @callback(

src/pages/observatoire.py

@@ -17,6 +17,7 @@ from dash import (
 )
 
 from src.cache import cache
+from src.db import query_marches, schema
 from src.figures import (
     DataTable,
     get_barchart_sources,
@@ -32,7 +33,6 @@ from src.figures import (
 from src.utils import (
     columns,
     departements,
-    df,
     df_acheteurs,
     df_titulaires,
     get_default_hidden_columns,
@@ -72,7 +72,7 @@ for code in departements.keys():
 
 OBSERVATOIRE_COLUMNS = [
     col
-    for col in df.columns
+    for col in schema.names()
     if col.startswith("acheteur")
     or col.startswith("titulaire")
     or col
@@ -663,7 +663,7 @@ def _normalize_filter_params(filter_params: dict) -> tuple:
 def _compute_dashboard_children(cache_key: tuple):
     filter_params = {k: (list(v) if isinstance(v, tuple) else v) for k, v in cache_key}
 
-    lff: pl.LazyFrame = df.lazy()
+    lff: pl.LazyFrame = query_marches().lazy()
     lff = prepare_dashboard_data(lff=lff, **filter_params)
 
     dff = lff.collect(engine="streaming")
@@ -788,7 +788,7 @@ def update_dashboard_cards(*filter_values):
     prevent_initial_call=True,
 )
 def download_observatoire(_n_clicks, filter_params, hidden_columns):
-    lff = prepare_dashboard_data(lff=df.lazy(), **(filter_params or {}))
+    lff = prepare_dashboard_data(lff=query_marches().lazy(), **(filter_params or {}))
 
     if hidden_columns:
         lff = lff.drop(hidden_columns)
@@ -879,7 +879,7 @@ def populate_preview_table(
     if not is_open:
         return (no_update,) * 9
 
-    lff = prepare_dashboard_data(lff=df.lazy(), **(filter_params or {}))
+    lff = prepare_dashboard_data(lff=query_marches().lazy(), **(filter_params or {}))
 
     return prepare_table_data(
         lff,

src/pages/tableau.py

@@ -19,17 +19,16 @@ from dash import (
     register_page,
 )
 
+from src.db import query_marches, schema
 from src.figures import DataTable, make_column_picker
 from src.utils import (
     columns,
-    df,
     filter_table_data,
     get_default_hidden_columns,
     invert_columns,
     logger,
     meta_content,
     prepare_table_data,
-    schema,
     sort_table_data,
 )
 
@@ -61,7 +60,7 @@ datatable = html.Div(
         filter_action="custom",
         sort_action="custom",
         hidden_columns=[],
-        columns=[{"id": col, "name": col} for col in df.columns],
+        columns=[{"id": col, "name": col} for col in schema.names()],
     ),
 )
 
@@ -128,7 +127,7 @@ layout = [
         ],
     ),
     dcc.Markdown(
-        f"Ce tableau contient tous les marchés attribués en France. Il vous permet d'appliquer un filtre sur une ou plusieurs colonnes, et ainsi produire la liste de marchés dont vous avez besoin (exemples : [marchés de voirie < 40 k€ en 2025](/tableau?filtres=%7Bacheteur_id%7D+icontains+24350013900189+%26%26+%7BdateNotification%7D+icontains+2025%2A+%26%26+%7Bmontant%7D+i%3C+40000+%26%26+%7Bobjet%7D+icontains+voirie&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2Cacheteur_departement_code%2CsourceDataset), [marchés > 500 k€ avec clause sociale attribués à des PME à plus de 100 km dans le Var](/tableau?filtres=%7Btitulaire_categorie%7D+icontains+PME+%26%26+%7Btitulaire_distance%7D+i%3E+100+%26%26+%7Bmontant%7D+i%3E+500000+%26%26+%7Bacheteur_departement_code%7D+icontains+83+%26%26+%7BconsiderationsSociales%7D+icontains+clause&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2CconsiderationsSociales%2Ctitulaire_distance%2Cacheteur_departement_code%2Ctitulaire_categorie%2CsourceDataset)). Par défaut seules quelques colonnes sont affichées, mais vous pouvez en afficher jusqu'à {str(df.width)} en cliquant sur le bouton **Choisir les colonnes**. Cet outil est assez puissant, je vous recommande de lire le mode d'emploi pour en tirer pleinement partie.",
+        f"Ce tableau contient tous les marchés attribués en France. Il vous permet d'appliquer un filtre sur une ou plusieurs colonnes, et ainsi produire la liste de marchés dont vous avez besoin (exemples : [marchés de voirie < 40 k€ en 2025](/tableau?filtres=%7Bacheteur_id%7D+icontains+24350013900189+%26%26+%7BdateNotification%7D+icontains+2025%2A+%26%26+%7Bmontant%7D+i%3C+40000+%26%26+%7Bobjet%7D+icontains+voirie&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2Cacheteur_departement_code%2CsourceDataset), [marchés > 500 k€ avec clause sociale attribués à des PME à plus de 100 km dans le Var](/tableau?filtres=%7Btitulaire_categorie%7D+icontains+PME+%26%26+%7Btitulaire_distance%7D+i%3E+100+%26%26+%7Bmontant%7D+i%3E+500000+%26%26+%7Bacheteur_departement_code%7D+icontains+83+%26%26+%7BconsiderationsSociales%7D+icontains+clause&colonnes=uid%2Cacheteur_id%2Cacheteur_nom%2Ctitulaire_id%2Ctitulaire_nom%2Cobjet%2Cmontant%2CdureeMois%2CdateNotification%2CconsiderationsSociales%2Ctitulaire_distance%2Cacheteur_departement_code%2Ctitulaire_categorie%2CsourceDataset)). Par défaut seules quelques colonnes sont affichées, mais vous pouvez en afficher jusqu'à {len(schema.names())} en cliquant sur le bouton **Choisir les colonnes**. Cet outil est assez puissant, je vous recommande de lire le mode d'emploi pour en tirer pleinement partie.",
         style={"maxWidth": "1000px"},
     ),
     html.Div(
@@ -188,7 +187,7 @@ layout = [
 
             ##### Afficher plus de colonnes
 
-            Par défaut, un nombre réduit de colonnes est affiché pour ne pas surcharger la page. Mais vous avez le choix parmi {str(df.width)} colonnes, ce serait dommage de vous limiter !
+            Par défaut, un nombre réduit de colonnes est affiché pour ne pas surcharger la page. Mais vous avez le choix parmi {len(schema.names())} colonnes, ce serait dommage de vous limiter !
 
             Pour afficher plus de colonnes, cliquez sur le bouton **Choisir les colonnes** et cochez les colonnes pour les afficher.
 
@@ -316,7 +315,7 @@ def update_table(href, page_current, page_size, filter_query, sort_by, data_time
     prevent_initial_call=True,
 )
 def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None):
-    lff: pl.LazyFrame = df.lazy()  # start from the original data
+    lff: pl.LazyFrame = query_marches().lazy()
 
     # Les colonnes masquées sont supprimées
     if hidden_columns:

src/pages/titulaire.py

@@ -15,6 +15,7 @@ from dash import (
     register_page,
 )
 
+from src.db import query_marches, schema
 from src.figures import (
     DataTable,
     get_distance_histogram,
@@ -24,7 +25,6 @@ from src.figures import (
 )
 from src.utils import (
     columns,
-    df,
     df_titulaires,
     filter_table_data,
     format_number,
@@ -69,7 +69,7 @@ datatable = html.Div(
         sort_action="custom",
         page_size=10,
         hidden_columns=[],
-        columns=[{"id": col, "name": col} for col in df.columns],
+        columns=[{"id": col, "name": col} for col in schema.names()],
     ),
 )
 
@@ -339,21 +339,18 @@ def update_titulaire_stats(data):
 )
 def get_titulaire_marches_data(url, titulaire_year: str) -> tuple:
     titulaire_siret = url.split("/")[-1]
-    lff = df.lazy()
-    lff = lff.filter(
-        (pl.col("titulaire_id") == titulaire_siret)
-        & (pl.col("titulaire_typeIdentifiant") == "SIRET")
-    )
+    lff = query_marches(
+        "titulaire_id = ? AND titulaire_typeIdentifiant = 'SIRET'",
+        (titulaire_siret,),
+    ).lazy()
     if titulaire_year and titulaire_year != "Toutes les années":
         lff = lff.filter(
             pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year)
         )
     lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True)
     lff = lff.fill_null("")
-
     dff: pl.DataFrame = lff.collect(engine="streaming")
     download_disabled, download_text, download_title = get_button_properties(dff.height)
-
     data = dff.to_dicts()
     return data, download_disabled, download_text, download_title
 

src/utils.py

@@ -4,15 +4,17 @@ import os
 import uuid
 from collections import OrderedDict
 from datetime import datetime, timedelta
-from time import localtime, sleep
+from time import localtime
 
 import polars as pl
 import polars.selectors as cs
 from dash import no_update
 from httpx import HTTPError, get, post
-from polars.exceptions import ComputeError
 from unidecode import unidecode
 
+from src.db import conn as duckdb_conn  # noqa: F401  (exposed for convenience)
+from src.db import get_cursor, query_marches, schema  # noqa: F401
+
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",
     level=logging.INFO,
@@ -231,59 +233,6 @@ def get_annuaire_data(siret: str) -> dict:
     return response
 
 
-def get_decp_data() -> pl.DataFrame:
-    # Chargement du fichier parquet
-    # Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment.
-    # On utilise polars pour la rapidité et la facilité de manipulation des données.
-
-    try:
-        logger.info(
-            f"Lecture du fichier parquet ({os.getenv('DATA_FILE_PARQUET_PATH')})..."
-        )
-        lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH"))
-    except ComputeError:
-        # Le fichier est probablement en cours de mise à jour
-        logger.info("Échec, nouvelle tentative dans 10s...")
-        sleep(10)
-        lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH"))
-
-    # Tri des marchés par date de notification
-    lff = lff.sort(by=["dateNotification", "uid"], descending=True, nulls_last=True)
-
-    # Uniquement les données actuelles, pas les anciennes versions de marchés
-    lff = lff.filter(pl.col("donneesActuelles")).drop("donneesActuelles")
-
-    # Convertir les colonnes booléennes en chaînes de caractères
-    lff = booleans_to_strings(lff)
-
-    # Mention pour les org dont on a pas le nom
-    for col in ["acheteur_nom", "titulaire_nom"]:
-        lff = lff.with_columns(
-            pl.when(pl.col(col).is_null())
-            .then(pl.lit("[Identifiant non reconnu dans la base INSEE]"))
-            .otherwise(pl.col(col))
-            .name.keep()
-        )
-
-    # Bizarrement je ne peux pas faire lff = lff.fill_null("") ici
-    # ça génère une erreur dans la page acheteur (acheteur_data.table) :
-    # AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import)
-
-    return lff.collect()
-
-
-def get_org_data(dff: pl.DataFrame, org_type: str) -> pl.DataFrame:
-    lff = dff.lazy()
-    lff = lff.select(
-        "uid",
-        cs.starts_with(org_type).exclude(
-            f"{org_type}_latitude", f"{org_type}_longitude"
-        ),
-    )
-    lff = lff.group_by(cs.starts_with(org_type)).len("Marchés")
-    return lff.collect()
-
-
 def get_statistics() -> dict:
     return (
         get(
@@ -634,7 +583,7 @@ def prepare_table_data(
     elif isinstance(data, pl.LazyFrame):
         lff = data
     else:
-        lff: pl.LazyFrame = df.lazy()  # start from the original data
+        lff: pl.LazyFrame = query_marches().lazy()
 
     # Application des filtres
     if filter_query:
@@ -888,29 +837,27 @@ def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dic
     return jsonld
 
 
-df: pl.DataFrame = get_decp_data()
-schema = df.collect_schema()
+# df_acheteurs / df_titulaires sont conservés en mémoire pour alimenter
+# la recherche sur la page d'accueil (autocomplétion, filtrage par sous-chaîne
+# à chaque frappe). Les colonnes reproduisent la sortie historique de
+# get_org_data(df, org_type).
+def _build_org_frame(org_type: str) -> pl.DataFrame:
+    org_cols = [
+        c
+        for c in schema.names()
+        if c.startswith(f"{org_type}_")
+        and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
+    ]
+    select_list = ", ".join(org_cols)
+    group_list = ", ".join(org_cols)
+    sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
+    return get_cursor().execute(sql).pl()
 
-df_acheteurs = get_org_data(df, "acheteur")
-df_titulaires = get_org_data(df, "titulaire")
-df_acheteurs_departement: pl.DataFrame = (
-    df_acheteurs.select(["acheteur_id", "acheteur_nom", "acheteur_departement_code"])
-    .unique()
-    .sort("acheteur_nom")
-)
-df_titulaires_departement: pl.DataFrame = (
-    df_titulaires.select(
-        ["titulaire_id", "titulaire_nom", "titulaire_departement_code"]
-    )
-    .unique()
-    .sort("titulaire_nom")
-)
-df_acheteurs_marches: pl.DataFrame = (
-    df.select("uid", "objet", "acheteur_id").unique().sort("acheteur_id")
-)
-df_titulaires_marches: pl.DataFrame = (
-    df.select("uid", "objet", "titulaire_id").unique().sort("titulaire_id")
-)
+
+df_acheteurs = _build_org_frame("acheteur")
+df_titulaires = _build_org_frame("titulaire")
+
+columns = schema.names()
 
 departements = get_departements()
 departements_geojson = get_departements_geojson()
@@ -926,4 +873,3 @@ meta_content = {
     ),
 }
 data_schema = get_data_schema()
-columns = df.columns

tests/conftest.py

@@ -1,5 +1,6 @@
 import datetime
 import os
+from pathlib import Path
 
 import polars as pl
 import pytest
@@ -41,12 +42,19 @@ def test_data():
             "titulaire_categorie": "PME",
         }
     ]
-    path = "tests/test.parquet"
-    path = os.path.abspath(path)
-    print(f"Writing test data to: {path}")  # <-- This will show you the real path
+    parquet_path = Path(os.path.abspath("tests/test.parquet"))
+    db_path = parquet_path.parent / "decp.duckdb"
+    print(f"Writing test data to: {parquet_path}")
 
-    pl.DataFrame(data).write_parquet("tests/test.parquet")
-    yield path
+    pl.DataFrame(data).write_parquet(parquet_path)
+
+    # Remove any stale DuckDB from a previous run so src.db rebuilds from
+    # the freshly-written parquet at import time.
+    for artifact in (db_path, db_path.with_suffix(".duckdb.tmp")):
+        if artifact.exists():
+            artifact.unlink()
+
+    yield str(parquet_path)
 
 
 def pytest_setup_options():

tests/test_db.py

@@ -0,0 +1,255 @@
+import datetime
+import os
+import time
+
+import polars as pl
+import pytest
+
+from src.db import should_rebuild
+
+
+@pytest.fixture
+def parquet_and_db(tmp_path, monkeypatch):
+    parquet = tmp_path / "source.parquet"
+    db = tmp_path / "decp.duckdb"
+    parquet.write_bytes(b"fake parquet content")
+    monkeypatch.delenv("REBUILD_DUCKDB", raising=False)
+    monkeypatch.delenv("DEVELOPMENT", raising=False)
+    return parquet, db
+
+
+def test_should_rebuild_when_db_missing(parquet_and_db):
+    parquet, db = parquet_and_db
+    assert should_rebuild(db, parquet) is True
+
+
+def test_should_rebuild_prod_when_parquet_newer(parquet_and_db, monkeypatch):
+    parquet, db = parquet_and_db
+    db.write_bytes(b"x")
+    parquet.touch()
+    now = time.time()
+    os.utime(db, (now, now))
+    os.utime(parquet, (now + 10, now + 10))
+    monkeypatch.setenv("DEVELOPMENT", "false")
+    assert should_rebuild(db, parquet) is True
+
+
+def test_should_not_rebuild_prod_when_parquet_older(parquet_and_db, monkeypatch):
+    parquet, db = parquet_and_db
+    parquet.touch()
+    db.write_bytes(b"x")
+    now = time.time()
+    os.utime(parquet, (now, now))
+    os.utime(db, (now + 10, now + 10))
+    monkeypatch.setenv("DEVELOPMENT", "false")
+    assert should_rebuild(db, parquet) is False
+
+
+def test_should_not_rebuild_dev_even_when_parquet_newer(parquet_and_db, monkeypatch):
+    parquet, db = parquet_and_db
+    db.write_bytes(b"x")
+    parquet.touch()
+    now = time.time()
+    os.utime(db, (now, now))
+    os.utime(parquet, (now + 10, now + 10))
+    monkeypatch.setenv("DEVELOPMENT", "true")
+    monkeypatch.delenv("REBUILD_DUCKDB", raising=False)
+    assert should_rebuild(db, parquet) is False
+
+
+def test_should_rebuild_dev_when_rebuild_forced(parquet_and_db, monkeypatch):
+    parquet, db = parquet_and_db
+    db.write_bytes(b"x")
+    parquet.touch()
+    now = time.time()
+    os.utime(db, (now, now))
+    os.utime(parquet, (now + 10, now + 10))
+    monkeypatch.setenv("DEVELOPMENT", "true")
+    monkeypatch.setenv("REBUILD_DUCKDB", "true")
+    assert should_rebuild(db, parquet) is True
+
+
+@pytest.fixture
+def built_db(tmp_path, monkeypatch):
+    """Build a DuckDB from a small Polars frame written as parquet."""
+    parquet_path = tmp_path / "source.parquet"
+    db_path = tmp_path / "decp.duckdb"
+
+    data = pl.DataFrame(
+        [
+            {
+                "uid": "1",
+                "id": "1",
+                "objet": "Travaux",
+                "acheteur_id": "A1",
+                "acheteur_nom": "Mairie",
+                "acheteur_departement_code": "75",
+                "titulaire_id": "T1",
+                "titulaire_nom": "Entreprise",
+                "titulaire_departement_code": "35",
+                "titulaire_typeIdentifiant": "SIRET",
+                "montant": 1000.0,
+                "dateNotification": datetime.date(2025, 1, 1),
+                "donneesActuelles": True,
+                "marcheInnovant": True,
+            },
+            {
+                "uid": "2",
+                "id": "2",
+                "objet": "Études",
+                "acheteur_id": "A1",
+                "acheteur_nom": "Mairie",
+                "acheteur_departement_code": "75",
+                "titulaire_id": "T2",
+                "titulaire_nom": None,
+                "titulaire_departement_code": "75",
+                "titulaire_typeIdentifiant": "SIRET",
+                "montant": 500.0,
+                "dateNotification": datetime.date(2024, 6, 1),
+                "donneesActuelles": True,
+                "marcheInnovant": False,
+            },
+            {
+                "uid": "3",
+                "id": "3",
+                "objet": "Ancien",
+                "acheteur_id": "A2",
+                "acheteur_nom": None,
+                "acheteur_departement_code": "13",
+                "titulaire_id": "T3",
+                "titulaire_nom": "Autre",
+                "titulaire_departement_code": "13",
+                "titulaire_typeIdentifiant": "SIRET",
+                "montant": 100.0,
+                "dateNotification": datetime.date(2023, 1, 1),
+                "donneesActuelles": False,  # must be filtered out
+                "marcheInnovant": False,
+            },
+        ]
+    )
+    data.write_parquet(parquet_path)
+    monkeypatch.setenv("DATA_FILE_PARQUET_PATH", str(parquet_path))
+
+    from src.db import build_database
+
+    build_database(db_path, parquet_path)
+    return db_path
+
+
+def test_build_filters_donnees_actuelles(built_db):
+    import duckdb
+
+    with duckdb.connect(str(built_db), read_only=True) as c:
+        rows = c.execute("SELECT uid FROM decp ORDER BY uid").fetchall()
+    assert [r[0] for r in rows] == ["1", "2"]
+
+
+def test_build_converts_booleans_to_oui_non(built_db):
+    import duckdb
+
+    with duckdb.connect(str(built_db), read_only=True) as c:
+        values = c.execute("SELECT marcheInnovant FROM decp ORDER BY uid").fetchall()
+    assert [v[0] for v in values] == ["oui", "non"]
+
+
+def test_build_replaces_null_org_names(built_db):
+    import duckdb
+
+    with duckdb.connect(str(built_db), read_only=True) as c:
+        titulaire_2 = c.execute(
+            "SELECT titulaire_nom FROM decp WHERE uid = '2'"
+        ).fetchone()
+    assert titulaire_2[0] == "[Identifiant non reconnu dans la base INSEE]"
+
+
+def test_build_creates_derived_tables(built_db):
+    import duckdb
+
+    with duckdb.connect(str(built_db), read_only=True) as c:
+        tables = {r[0] for r in c.execute("SHOW TABLES").fetchall()}
+    assert {
+        "decp",
+        "acheteurs_marches",
+        "titulaires_marches",
+        "acheteurs_departement",
+        "titulaires_departement",
+    } <= tables
+
+
+def test_query_marches_returns_polars_frame(built_db, monkeypatch):
+    monkeypatch.setenv(
+        "DATA_FILE_PARQUET_PATH", str(built_db.parent / "source.parquet")
+    )
+    # Force src.db to load pointing at this test DB.
+    import importlib
+
+    import src.db
+
+    importlib.reload(src.db)
+    from src.db import query_marches
+
+    frame = query_marches("acheteur_id = ?", ("A1",))
+    assert isinstance(frame, pl.DataFrame)
+    assert frame.height == 2
+    assert set(frame["uid"].to_list()) == {"1", "2"}
+
+
+def test_concurrent_build_serialized(tmp_path):
+    """Multiple threads calling _ensure_database must serialize via flock.
+
+    Only one should actually build; others wait, see the fresh DB, and skip.
+    No tmp file should leak. No exceptions should occur.
+    """
+    import fcntl
+    import threading
+
+    import src.db as db
+
+    # Set up source parquet
+    parquet_path = tmp_path / "src.parquet"
+    df = pl.DataFrame(
+        {
+            "uid": ["A"],
+            "donneesActuelles": [True],
+            "dateNotification": ["2024-01-01"],
+            "objet": ["Test"],
+            "acheteur_id": ["a1"],
+            "acheteur_nom": ["A1"],
+            "titulaire_id": ["t1"],
+            "titulaire_nom": ["T1"],
+            "acheteur_departement_code": ["75"],
+            "titulaire_departement_code": ["75"],
+            "montant": [1000.0],
+            "dureeMois": [12],
+        }
+    )
+    df.write_parquet(parquet_path)
+
+    db_path = tmp_path / "decp.duckdb"
+    lock_path = db_path.with_suffix(".duckdb.lock")
+    tmp_path_artifact = db_path.with_suffix(".duckdb.tmp")
+
+    errors: list[BaseException] = []
+
+    def worker():
+        try:
+            # Mirror the locking logic in _ensure_database
+            with open(lock_path, "w") as lf:
+                fcntl.flock(lf.fileno(), fcntl.LOCK_EX)
+                try:
+                    if db.should_rebuild(db_path, parquet_path):
+                        db.build_database(db_path, parquet_path)
+                finally:
+                    fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
+        except BaseException as exc:
+            errors.append(exc)
+
+    threads = [threading.Thread(target=worker) for _ in range(3)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    assert errors == []
+    assert db_path.exists()
+    assert not tmp_path_artifact.exists()