colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

Merge branch 'main' into feature/78_api

Browse Source
This commit is contained in:
Colin Maudry
2026-06-12 09:33:59 +02:00
parent 8dc5287918 c304f05d85
commit 1aea71be7f
15 changed files with 2845 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.py
+15 -8
View File
@@ -6,7 +6,7 @@ import pandas # noqa: F401 # eager import: avoid plotly's lazy-import race acr
import tomllib
from dash import Dash, Input, Output, State, dcc, html, page_container, page_registry
from dotenv import load_dotenv
from flask import Response
from flask import Flask, Response
from src.utils import DEVELOPMENT
from src.utils.cache import cache
@@ -27,12 +27,10 @@ META_TAGS = [
if DEVELOPMENT:
META_TAGS.append({"name": "robots", "content": "noindex"})
app: Dash = Dash(
title="decp.info",
use_pages=True,
compress=True,
meta_tags=META_TAGS,
)
# Le cache doit être initialisé AVANT la construction de Dash : `use_pages=True`
# importe les modules de pages pendant l'instanciation, et certains appellent des
# fonctions memoizées (@cache.memoize) dès l'import (ex. tableau.py).
server = Flask(__name__)
cache_dir = os.getenv("CACHE_DIR", "/tmp/decp-cache")
@@ -40,7 +38,7 @@ if os.path.exists(cache_dir):
rmtree(cache_dir)
cache.init_app(
app.server,
server,
config={
"CACHE_TYPE": "FileSystemCache",
"CACHE_DIR": cache_dir,
@@ -51,6 +49,14 @@ cache.init_app(
},
)
app: Dash = Dash(
server=server,
title="decp.info",
use_pages=True,
compress=True,
meta_tags=META_TAGS,
)
from src.api import init_api # noqa: E402 # inline: src.db.conn must be ready first
init_api(app.server)
@@ -73,6 +79,7 @@ def sitemap():
"/observatoire",
"/tableau",
"/a-propos",
"/etapes",
]
xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
xml += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
src/assets/css/style.css
+215
View File
@@ -546,3 +546,218 @@ input[type="number"]::-webkit-inner-spin-button {
input[type="number"] {
-moz-appearance: textfield;
}
/* ===== Page /etapes : graphique données par étape et par seuil ===== */
.etapes-chart-scroll {
overflow-x: auto;
margin: 1rem 0;
}
.etapes-chart {
min-width: 720px;
background: #fff;
border: 1px solid #d0d5dd;
border-radius: 8px;
overflow: hidden;
font-size: 13px;
display: grid;
grid-template-columns: 150px repeat(5, 1fr);
}
.etapes-corner {
border-bottom: 2px solid #344054;
}
.etapes-xhead {
grid-column: 2 / -1;
display: grid;
grid-template-columns: repeat(5, 1fr);
border-bottom: 2px solid #344054;
}
.etapes-xcell {
text-align: center;
padding: 6px 2px;
font-size: 11px;
color: #475467;
border-left: 1px dashed #d0d5dd;
}
.etapes-xcell strong {
display: block;
color: #101828;
font-size: 12px;
}
.etapes-stage {
padding: 14px 10px;
font-weight: 600;
color: #101828;
border-bottom: 1px solid #eaecf0;
display: flex;
align-items: center;
}
.etapes-stage small {
font-weight: 400;
color: #667085;
}
.etapes-lane {
grid-column: 2 / -1;
position: relative;
border-bottom: 1px solid #eaecf0;
min-height: 52px;
}
.etapes-segs {
position: absolute;
inset: 0;
display: grid;
grid-template-columns: repeat(5, 1fr);
}
.etapes-segs > div {
border-left: 1px dashed #eaecf0;
}
.etapes-bar {
position: absolute;
top: 9px;
height: 32px;
border-radius: 6px;
color: #fff;
font-size: 11px;
font-weight: 600;
display: flex;
align-items: center;
padding: 0 10px;
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.12);
white-space: nowrap;
overflow: hidden;
cursor: pointer;
transition: filter 0.15s, box-shadow 0.15s;
}
.etapes-bar:hover {
filter: brightness(1.12);
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.22);
}
.etapes-empty {
color: #98a2b3;
font-style: italic;
padding: 14px;
display: flex;
align-items: center;
}
.etapes-note {
margin-top: 8px;
color: #667085;
font-size: 13px;
}
/* --- Vue mobile (liste par étape) : masquée par défaut --- */
.etapes-mobile {
display: none;
margin: 1rem 0;
}
.etapes-m-block {
border: 1px solid #d0d5dd;
border-radius: 8px;
margin-bottom: 12px;
overflow: hidden;
}
.etapes-m-header {
display: flex;
align-items: center;
justify-content: space-between;
background: #f9fafb;
border-bottom: 1px solid #eaecf0;
}
.etapes-m-stage {
margin: 0;
padding: 10px 12px;
font-size: 15px;
color: #101828;
}
.etapes-m-link {
background: none;
border: none;
color: #1570ef;
font-size: 12px;
font-weight: 600;
cursor: pointer;
padding: 0 12px;
white-space: nowrap;
}
.etapes-m-link:hover {
text-decoration: underline;
}
.etapes-m-item {
display: flex;
align-items: baseline;
gap: 8px;
padding: 8px 12px;
border-bottom: 1px solid #f2f4f7;
font-size: 13px;
}
.etapes-m-item:last-child {
border-bottom: none;
}
.etapes-m-item i {
width: 12px;
height: 12px;
border-radius: 3px;
flex: 0 0 auto;
position: relative;
top: 2px;
}
.etapes-m-label {
font-weight: 600;
color: #101828;
}
.etapes-m-seuil {
color: #667085;
}
.etapes-m-empty {
color: #98a2b3;
font-style: italic;
}
.etapes-detail {
margin: 1rem 0;
padding: 1rem 1.25rem;
border: 1px solid #d0d5dd;
border-radius: 8px;
background: #f9fafb;
}
.etapes-detail:empty {
display: none;
}
/* --- Bascule desktop / mobile au point de rupture 768 px --- */
@media (max-width: 768px) {
.etapes-chart-scroll {
display: none;
}
.etapes-mobile {
display: block;
}
}
src/db.py
+16 -11
View File
@@ -8,27 +8,31 @@ import polars as pl
import polars.selectors as cs
from polars.exceptions import ComputeError
from src.utils import logger
from src.utils import get_last_modified, logger
def should_rebuild(db_path: Path, parquet_path: Path) -> bool:
def should_rebuild(db_path: Path, parquet_path: str) -> bool:
db_path = Path(db_path)
parquet_path = Path(parquet_path)
if not db_path.exists():
return True
dev = os.getenv("DEVELOPMENT", "False").lower() == "true"
force = os.getenv("REBUILD_DUCKDB", "False").lower() == "true"
if dev and not force:
return False
return parquet_path.stat().st_mtime > db_path.stat().st_mtime
last_modified: float = get_last_modified(parquet_path)
return last_modified > db_path.stat().st_mtime
def _load_source_frame(parquet_path: Path) -> pl.DataFrame:
def _load_source_frame() -> pl.DataFrame:
"""Read the source parquet and apply the row-level transforms.
Kept here (not in utils.py) so src.db has no dependency on utils.
Mirrors the behavior previously in utils.get_decp_data().
"""
parquet_path: str = os.getenv("DATA_FILE_PARQUET_PATH", "")
if not (parquet_path.startswith("http")):
assert os.path.exists(parquet_path)
try:
lff: pl.LazyFrame = pl.scan_parquet(str(parquet_path))
except ComputeError:
@@ -58,20 +62,21 @@ def _load_source_frame(parquet_path: Path) -> pl.DataFrame:
return lff.collect()
def build_database(db_path: Path, parquet_path: Path) -> None:
def build_database(db_path: Path) -> None:
"""Build the DuckDB database atomically under an exclusive lock.
Caller MUST hold the fcntl.flock on the .lock file.
"""
db_path = Path(db_path)
parquet_path = Path(parquet_path)
tmp_path = db_path.with_suffix(".duckdb.tmp")
staging_parquet = db_path.with_suffix(".staging.parquet")
if tmp_path.exists():
tmp_path.unlink()
logger.info(f"Construction de la base DuckDB à partir de {parquet_path}...")
frame = _load_source_frame(parquet_path)
logger.info(
f"Construction de la base DuckDB à partir de {os.getenv('DATA_FILE_PARQUET_PATH', '')}..."
)
frame = _load_source_frame()
# Write transformed frame as parquet so DuckDB can read it natively
# (avoids pyarrow dependency for the Polars→DuckDB handoff)
@@ -111,13 +116,13 @@ def build_database(db_path: Path, parquet_path: Path) -> None:
def _ensure_database() -> Path:
db_path = Path(os.getenv("DUCKDB_PATH", "./decp.duckdb"))
parquet_path = Path(os.getenv("DATA_FILE_PARQUET_PATH"))
parquet_path = os.getenv("DATA_FILE_PARQUET_PATH", "")
lock_path = db_path.with_suffix(".duckdb.lock")
with open(lock_path, "w") as lock_fd:
fcntl.flock(lock_fd, fcntl.LOCK_EX)
if should_rebuild(db_path, parquet_path):
build_database(db_path, parquet_path)
build_database(db_path)
else:
logger.debug("Base de données déjà disponible et à jour.")
return db_path
src/pages/arbre/liste_marches_org.py
+7 -3
View File
@@ -2,6 +2,7 @@ import polars as pl
from dash import Input, Output, callback, dcc, html, register_page
from src.db import get_cursor
from src.utils import logger
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
NAME = "Liste des marchés publics"
@@ -27,9 +28,12 @@ def make_org_nom_verbe(org_type, org_id) -> tuple:
def get_title(code, org_type, org_id):
org_nom, verbe = make_org_nom_verbe(org_type, org_id)
return f"Marchés publics {verbe} par {org_nom} | decp.info"
if org_type:
org_nom, verbe = make_org_nom_verbe(org_type, org_id)
return f"Marchés publics {verbe} par {org_nom} | decp.info"
else:
logger.warning(f"Pas de org_type pour org_id: {org_id}")
return "Marchés publics | decp.info"
def get_description(code, org_type, org_id):
src/pages/etapes.py
+297
View File
@@ -0,0 +1,297 @@
from dash import Input, Output, State, callback, ctx, dcc, html, register_page
from src.utils.seo import META_CONTENT
NAME = "Quelles données pour quelles étapes et quels seuils dans les marchés publics ?"
register_page(
__name__,
path="/etapes",
title=f"{NAME} | decp.info",
name="Étapes et données",
description=(
"À chaque étape d'un marché public (programmation, publicité, "
"attribution), quelles données sont publiées et à partir de quel "
"seuil : DECP, BOAMP, JOUE, journaux d'annonces légales, Approch."
),
image_url=META_CONTENT["image_url"],
)
# Contenu des fiches — à rédiger en Markdown.
# Clés barres : "bar-approch", "bar-jal", "bar-boamp", "bar-joue-marche",
# "bar-decp", "bar-joue-attribution"
# Clés étapes (mobile) : "stage-programmation", "stage-publicite",
# "stage-attribution", "stage-contrat", "stage-paiement"
ALL_CONTENT: dict[str, str | None] = {
"bar-approch": None,
"bar-jal": None,
"bar-boamp": None,
"bar-joue-marche": None,
"bar-decp": None,
"bar-joue-attribution": None,
"stage-programmation": None,
"stage-publicite": None,
"stage-attribution": None,
"stage-contrat": None,
"stage-paiement": None,
}
_BAR_IDS = [
"bar-approch",
"bar-jal",
"bar-boamp",
"bar-joue-marche",
"bar-decp",
"bar-joue-attribution",
]
_STAGE_IDS = [
"stage-programmation",
"stage-publicite",
"stage-attribution",
"stage-contrat",
"stage-paiement",
]
def _lane(*bars):
"""Une ligne d'étape : fond segmenté en 5 + barres positionnées."""
return html.Div(
className="etapes-lane",
children=[
html.Div(
className="etapes-segs",
children=[html.Div() for _ in range(5)],
),
*bars,
],
)
def _bar(label, color, style, bar_id=None):
base = {"backgroundColor": color}
base.update(style)
props = {"className": "etapes-bar", "style": base}
if bar_id is not None:
props["id"] = bar_id
props["n_clicks"] = 0
return html.Div(label, **props)
def build_chart():
return html.Div(
className="etapes-chart-scroll",
children=html.Div(
className="etapes-chart",
children=[
# En-tête : coin vide + 5 marqueurs de seuils
html.Div(className="etapes-corner"),
html.Div(
className="etapes-xhead",
children=[
html.Div("0 €", className="etapes-xcell"),
html.Div(
[html.Strong("40 000 €"), "seuil DECP"],
className="etapes-xcell",
),
html.Div(
[html.Strong("90 000 €"), "publicité"],
className="etapes-xcell",
),
html.Div(
[html.Strong("140 k€ / 216 k€"), "seuils formalisés (UE)"],
className="etapes-xcell",
),
html.Div(
[html.Strong("5,404 M€"), "travaux (UE)"],
className="etapes-xcell",
),
],
),
# Programmation
html.Div("Programmation", className="etapes-stage"),
_lane(
_bar(
"Approch — sourcing / préinformation (non réglementaire)",
"#7c5cff",
{"left": "2%", "right": "2%"},
bar_id="bar-approch",
),
),
# Publicité (appel d'offres)
html.Div(["Publicité"], className="etapes-stage"),
_lane(
_bar(
"JAL",
"#f79009",
{"left": "40%", "right": "40%", "top": "6px", "height": "20px"},
bar_id="bar-jal",
),
_bar(
"BOAMP",
"#1570ef",
{"left": "40%", "right": "2%", "top": "28px", "height": "20px"},
bar_id="bar-boamp",
),
_bar(
"JOUE — avis de marché",
"#0e9384",
{"left": "60%", "right": "2%", "top": "6px", "height": "20px"},
bar_id="bar-joue-marche",
),
),
# Attribution
html.Div("Attribution", className="etapes-stage"),
_lane(
_bar(
"DECP — données essentielles",
"#12b76a",
{"left": "20%", "right": "2%", "top": "6px", "height": "20px"},
bar_id="bar-decp",
),
_bar(
"JOUE — avis d'attribution",
"#0e9384",
{"left": "60%", "right": "2%", "top": "28px", "height": "20px"},
bar_id="bar-joue-attribution",
),
),
# Contrat (vide)
html.Div("Contrat", className="etapes-stage"),
html.Div(
"— aucune donnée publiée aujourd'hui —",
className="etapes-lane etapes-empty",
),
# Paiement (vide)
html.Div("Paiement", className="etapes-stage"),
html.Div(
"— aucune donnée publiée aujourd'hui —",
className="etapes-lane etapes-empty",
),
],
),
)
# Données par étape, partagées par la vue mobile.
# Chaque tuple : (libellé étape, id CSS, [(libellé, couleur, plage seuils)]).
STAGES_MOBILE = [
(
"Programmation",
"stage-programmation",
[
("Approch", "#7c5cff", "tous montants — publication non réglementaire"),
],
),
(
"Publicité (appel d'offres)",
"stage-publicite",
[
("JAL", "#f79009", "de 90 000 € au seuil formalisé"),
("BOAMP", "#1570ef", "à partir de 90 000 €"),
(
"JOUE — avis de marché",
"#0e9384",
"à partir des seuils formalisés (140 k€ / 216 k€)",
),
],
),
(
"Attribution",
"stage-attribution",
[
("DECP — données essentielles", "#12b76a", "à partir de 40 000 €"),
("JOUE — avis d'attribution", "#0e9384", "à partir des seuils formalisés"),
],
),
("Contrat", "stage-contrat", []),
("Paiement", "stage-paiement", []),
]
def build_mobile():
blocks = []
for stage, stage_id, items in STAGES_MOBILE:
if items:
children = [
html.Div(
[
html.I(style={"backgroundColor": color}),
html.Span(label, className="etapes-m-label"),
html.Span(seuil, className="etapes-m-seuil"),
],
className="etapes-m-item",
)
for label, color, seuil in items
]
else:
children = [
html.Div(
"aucune donnée publiée aujourd'hui",
className="etapes-m-item etapes-m-empty",
)
]
blocks.append(
html.Div(
[
html.Div(
[
html.H4(stage, className="etapes-m-stage"),
html.Button(
"Voir fiche →",
id=stage_id,
n_clicks=0,
className="etapes-m-link",
),
],
className="etapes-m-header",
),
*children,
],
className="etapes-m-block",
)
)
return html.Div(blocks, className="etapes-mobile")
layout = html.Div(
className="container",
children=[
html.H2(NAME),
dcc.Markdown(
"Un marché public passe par plusieurs étapes. À chacune, des "
"données peuvent être publiées — selon le montant du marché et "
"des obligations réglementaires. Ce graphique situe les "
"principales publications de données par **étape** (de haut en "
"bas) et par **seuil** (de gauche à droite, en euros hors taxes)."
),
build_chart(),
build_mobile(),
dcc.Store(id="etapes-selected", data=None),
html.Div(id="etapes-detail", className="etapes-detail"),
dcc.Markdown(
"**À noter :** l'axe horizontal n'est pas linéaire — les seuils "
"sont espacés régulièrement pour rester lisibles. Les étapes "
"*Contrat* et *Paiement* n'ont aujourd'hui aucune donnée publiée "
"en open data.",
className="etapes-note",
),
],
)
@callback(
Output("etapes-detail", "children"),
Output("etapes-selected", "data"),
[Input(id_, "n_clicks") for id_ in _BAR_IDS + _STAGE_IDS],
State("etapes-selected", "data"),
prevent_initial_call=True,
)
def _show_detail(*args):
current = args[-1]
triggered = ctx.triggered_id
if triggered == current:
return None, None
content = ALL_CONTENT.get(triggered)
if content is None:
return dcc.Markdown(f"*Fiche en cours de rédaction.* {triggered}"), triggered
return dcc.Markdown(content), triggered
src/pages/marche.py
+2 -1
View File
@@ -109,7 +109,7 @@ def update_marche_info(marche, titulaires):
column_object = DATA_SCHEMA.get(col)
column_name = column_object.get("title") if column_object else col
if col in marche:
if marche and col in marche:
if col == "acheteur_nom":
value = html.A(
href=f"/acheteurs/{marche['acheteur_id']}",
@@ -134,6 +134,7 @@ def update_marche_info(marche, titulaires):
"considerationsSociales",
"considerationsEnvironnementales",
]
and col in marche
and "," in marche[col]
):
col_values = marche[col].split(", ")
src/pages/tableau.py
+2 -2
View File
@@ -21,7 +21,7 @@ from dash import (
from src.db import query_marches, schema
from src.figures import DataTable, make_column_picker
from src.utils import logger
from src.utils import get_last_modified, logger
from src.utils.seo import META_CONTENT
from src.utils.table import (
COLUMNS,
@@ -33,7 +33,7 @@ from src.utils.table import (
)
from src.utils.tracking import track_search
update_date_timestamp = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH"))
update_date_timestamp = get_last_modified(os.getenv("DATA_FILE_PARQUET_PATH", ""))
update_date = datetime.fromtimestamp(update_date_timestamp).strftime("%d/%m/%Y")
update_date_iso = datetime.fromtimestamp(update_date_timestamp).isoformat()
src/utils/__init__.py
+24
View File
@@ -1,5 +1,29 @@
import logging
import os
from datetime import datetime
from pathlib import Path
import httpx
from src.utils.cache import cache
@cache.memoize()
def get_last_modified(parquet_path: str) -> float:
logger.info("Récupération de la date de modification des données...")
logging.getLogger("httpx").setLevel("WARNING")
if parquet_path.startswith("http"):
last_modified = httpx.head(
url=parquet_path,
follow_redirects=True,
).headers["last-modified"]
last_modified = datetime.strptime(last_modified, "%a, %d %b %Y %X %Z").strftime(
"%s"
)
return float(last_modified)
parquet_local_path = Path(parquet_path)
return parquet_local_path.stat().st_mtime
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
src/utils/data.py
+30 -17
View File
@@ -2,7 +2,9 @@ import json
import logging
import os
from collections import OrderedDict
from pathlib import Path
import httpx
import polars as pl
from httpx import HTTPError, get
@@ -51,28 +53,39 @@ def get_departements_geojson() -> dict:
return geojson
def get_departement_region(code_postal):
if code_postal > "97000":
code_departement = code_postal[:3]
else:
code_departement = code_postal[:2]
nom_departement = DEPARTEMENTS[code_departement]["departement"]
nom_region = DEPARTEMENTS[code_departement]["region"]
return code_departement, nom_departement, nom_region
def get_departement_region(code_postal: str | None):
if code_postal:
if code_postal > "97000":
code_departement = code_postal[:3]
else:
code_departement = code_postal[:2]
nom_departement = DEPARTEMENTS[code_departement]["departement"]
nom_region = DEPARTEMENTS[code_departement]["region"]
return code_departement, nom_departement, nom_region
return "", "", ""
def get_data_schema() -> dict:
# Récupération du schéma des données tabulaires
path = os.getenv("DATA_SCHEMA_PATH")
if path.startswith("http"):
original_schema: dict = get(
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
).json()
elif os.path.exists(path):
with open(path) as f:
url = os.getenv("DATA_SCHEMA_PATH")
local_path = Path(os.getenv("DATA_SCHEMA_LOCAL", ""))
original_schema = {}
if url:
try:
original_schema: dict = get(url, follow_redirects=True).json()
except (
httpx.ReadTimeout,
httpx.ReadError,
httpx.ConnectError,
httpx.ConnectTimeout,
):
logger.error(f"Erreur HTTP lors de la récupération du schéma ({url})")
if os.path.exists(local_path) and original_schema == {}:
with open(local_path) as f:
original_schema: dict = json.load(f)
else:
raise Exception(f"Chemin vers le schéma invalide: {path}")
logger.info(f"Utilisation du schéma local ({local_path})")
new_schema = OrderedDict()