Merge branch 'dev' into feature/73_compte_utilisateur
This commit is contained in:
Colin Maudry
@@ -0,0 +1,34 @@
|
||||
from flask_smorest import Api
|
||||
|
||||
from src.api import routes
|
||||
|
||||
|
||||
def init_api(server) -> None:
|
||||
"""Enregistre le blueprint d'API privée sur le serveur Flask."""
|
||||
server.config.setdefault("API_TITLE", "decp.info API")
|
||||
server.config.setdefault("API_VERSION", "v1")
|
||||
server.config.setdefault("OPENAPI_VERSION", "3.0.3")
|
||||
server.config.setdefault("OPENAPI_URL_PREFIX", "/api/v1")
|
||||
server.config.setdefault("OPENAPI_JSON_PATH", "openapi.json")
|
||||
server.config.setdefault("OPENAPI_SWAGGER_UI_PATH", "swagger")
|
||||
server.config.setdefault(
|
||||
"OPENAPI_SWAGGER_UI_URL",
|
||||
"https://cdn.jsdelivr.net/npm/swagger-ui-dist/",
|
||||
)
|
||||
server.config.setdefault(
|
||||
"API_SPEC_OPTIONS",
|
||||
{
|
||||
"components": {
|
||||
"securitySchemes": {"BearerAuth": {"type": "http", "scheme": "bearer"}}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
api = Api(server)
|
||||
api.register_blueprint(routes.bp)
|
||||
|
||||
import os
|
||||
|
||||
from src.api import tracking
|
||||
|
||||
tracking.start_worker(os.environ["USERS_DB_PATH"])
|
||||
@@ -0,0 +1,35 @@
|
||||
import os
|
||||
from functools import wraps
|
||||
|
||||
from flask import abort, g, jsonify, make_response, request
|
||||
|
||||
from src.api import tokens_db
|
||||
|
||||
API_AUTH_DISABLED = os.getenv("API_AUTH_DISABLED", "False").lower() == "true"
|
||||
|
||||
|
||||
def _abort_401(message: str):
|
||||
resp = make_response(jsonify({"message": message}), 401)
|
||||
abort(resp)
|
||||
|
||||
|
||||
def require_token(fn):
|
||||
@wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not API_AUTH_DISABLED:
|
||||
header = request.headers.get("Authorization", "")
|
||||
if not header.startswith("Bearer "):
|
||||
_abort_401("missing_token")
|
||||
token = header[len("Bearer ") :].strip()
|
||||
if not token:
|
||||
_abort_401("missing_token")
|
||||
db_path = os.environ["USERS_DB_PATH"]
|
||||
row = tokens_db.get_token_by_plaintext(db_path, token)
|
||||
if row is None:
|
||||
_abort_401("invalid_token")
|
||||
if row["revoked_at"] is not None:
|
||||
_abort_401("revoked_token")
|
||||
g.token_id = row["id"]
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
@@ -0,0 +1,195 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
|
||||
import polars as pl
|
||||
|
||||
OPERATORS = {
|
||||
"exact",
|
||||
"contains",
|
||||
"notcontains",
|
||||
"differs",
|
||||
"less",
|
||||
"greater",
|
||||
"strictly_less",
|
||||
"strictly_greater",
|
||||
"in",
|
||||
"notin",
|
||||
"isnull",
|
||||
"isnotnull",
|
||||
"sort",
|
||||
}
|
||||
|
||||
RESERVED_PARAMS = {"page", "page_size", "columns", "count_results"}
|
||||
|
||||
AGGREGATORS = {"groupby", "count", "sum", "avg", "min", "max"}
|
||||
AGG_SQL = {"count": "COUNT", "sum": "SUM", "avg": "AVG", "min": "MIN", "max": "MAX"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class AggregationSpec:
|
||||
select_sql: str
|
||||
group_by_sql: str | None
|
||||
|
||||
|
||||
def parse_aggregators(
|
||||
args: list[tuple[str, str]], schema: pl.Schema
|
||||
) -> "AggregationSpec | None":
|
||||
"""Détecte les drapeaux d'agrégation (`col__groupby`, `col__count`, ...).
|
||||
|
||||
Retourne None si aucun agrégateur. Sinon, construit les fragments SQL
|
||||
`select_sql` et `group_by_sql` (noms de colonnes validés contre le schéma).
|
||||
"""
|
||||
group_cols: list[str] = []
|
||||
aggregates: list[tuple[str, str]] = [] # (operator, column)
|
||||
has_agg = False
|
||||
|
||||
for key, _ in args:
|
||||
parsed = _split_key(key)
|
||||
if not parsed:
|
||||
continue
|
||||
col, op = parsed
|
||||
if op not in AGGREGATORS:
|
||||
continue
|
||||
has_agg = True
|
||||
if col not in schema:
|
||||
raise FilterError(f"Colonne inconnue : {col!r}", field=key)
|
||||
if op == "groupby":
|
||||
group_cols.append(col)
|
||||
else:
|
||||
aggregates.append((op, col))
|
||||
|
||||
if not has_agg:
|
||||
return None
|
||||
|
||||
select_parts = [f'"{c}"' for c in group_cols]
|
||||
for op, col in aggregates:
|
||||
select_parts.append(f'{AGG_SQL[op]}("{col}") AS "{col}__{op}"')
|
||||
|
||||
group_by_sql = ", ".join(f'"{c}"' for c in group_cols) if group_cols else None
|
||||
return AggregationSpec(
|
||||
select_sql=", ".join(select_parts), group_by_sql=group_by_sql
|
||||
)
|
||||
|
||||
|
||||
class FilterError(ValueError):
|
||||
def __init__(self, message: str, field: str | None = None):
|
||||
super().__init__(message)
|
||||
self.field = field
|
||||
|
||||
|
||||
def _coerce(value: str, dtype: pl.DataType, key: str):
|
||||
if dtype == pl.String:
|
||||
return value
|
||||
if dtype.is_integer():
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
raise FilterError(f"Valeur entière attendue, reçu {value!r}", field=key)
|
||||
if dtype.is_float():
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
raise FilterError(f"Valeur décimale attendue, reçu {value!r}", field=key)
|
||||
if dtype == pl.Date:
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError:
|
||||
raise FilterError(
|
||||
f"Date ISO 8601 attendue (YYYY-MM-DD), reçu {value!r}", field=key
|
||||
)
|
||||
if dtype == pl.Datetime:
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
raise FilterError(f"Datetime ISO 8601 attendu, reçu {value!r}", field=key)
|
||||
return value
|
||||
|
||||
|
||||
def _split_key(key: str) -> tuple[str, str] | None:
|
||||
if "__" not in key:
|
||||
return None
|
||||
col, _, op = key.rpartition("__")
|
||||
if not col or not op:
|
||||
return None
|
||||
return col, op
|
||||
|
||||
|
||||
def build_where(
|
||||
args: list[tuple[str, str]], schema: pl.Schema
|
||||
) -> tuple[str, list, str | None]:
|
||||
"""Parse query params into (where_sql, params, order_by_sql).
|
||||
|
||||
args: list of (key, value) tuples preserving URL order (Werkzeug MultiDict
|
||||
preserves insertion order on `request.args.items(multi=True)`).
|
||||
"""
|
||||
where_parts: list[str] = []
|
||||
params: list = []
|
||||
order_parts: list[str] = []
|
||||
|
||||
for key, value in args:
|
||||
if key in RESERVED_PARAMS:
|
||||
continue
|
||||
|
||||
parsed = _split_key(key)
|
||||
if not parsed:
|
||||
raise FilterError(f"Paramètre non reconnu : {key}", field=key)
|
||||
col, op = parsed
|
||||
|
||||
if op in AGGREGATORS:
|
||||
continue
|
||||
|
||||
if op not in OPERATORS:
|
||||
raise FilterError(f"Opérateur inconnu : __{op}", field=key)
|
||||
|
||||
if col not in schema:
|
||||
raise FilterError(f"Colonne inconnue : {col!r}", field=key)
|
||||
|
||||
if op == "sort":
|
||||
direction = value.lower()
|
||||
if direction not in ("asc", "desc"):
|
||||
raise FilterError(
|
||||
f"Tri attendu 'asc' ou 'desc', reçu {value!r}", field=key
|
||||
)
|
||||
order_parts.append(f'"{col}" {direction.upper()}')
|
||||
continue
|
||||
|
||||
if op in ("isnull", "isnotnull"):
|
||||
sql = "IS NULL" if op == "isnull" else "IS NOT NULL"
|
||||
where_parts.append(f'"{col}" {sql}')
|
||||
continue
|
||||
|
||||
dtype = schema[col]
|
||||
|
||||
if op in ("in", "notin"):
|
||||
values = [_coerce(v.strip(), dtype, key) for v in value.split(",")]
|
||||
placeholders = ",".join(["?"] * len(values))
|
||||
sql_op = "IN" if op == "in" else "NOT IN"
|
||||
where_parts.append(f'"{col}" {sql_op} ({placeholders})')
|
||||
params.extend(values)
|
||||
continue
|
||||
|
||||
v = _coerce(value, dtype, key)
|
||||
|
||||
op_sql = {
|
||||
"exact": "=",
|
||||
"less": "<=",
|
||||
"greater": ">=",
|
||||
"strictly_less": "<",
|
||||
"strictly_greater": ">",
|
||||
}
|
||||
if op in op_sql:
|
||||
where_parts.append(f'"{col}" {op_sql[op]} ?')
|
||||
params.append(v)
|
||||
elif op == "contains":
|
||||
where_parts.append(f'"{col}" LIKE ?')
|
||||
params.append(f"%{v}%")
|
||||
elif op == "notcontains":
|
||||
where_parts.append(f'"{col}" NOT LIKE ?')
|
||||
params.append(f"%{v}%")
|
||||
elif op == "differs":
|
||||
where_parts.append(f'"{col}" IS DISTINCT FROM ?')
|
||||
params.append(v)
|
||||
|
||||
where_sql = " AND ".join(where_parts) if where_parts else "TRUE"
|
||||
order_sql = ", ".join(order_parts) if order_parts else None
|
||||
return where_sql, params, order_sql
|
||||
@@ -0,0 +1,236 @@
|
||||
from flask import g, request
|
||||
from flask_smorest import Blueprint, abort
|
||||
|
||||
from src.api import tracking
|
||||
from src.api.auth import require_token
|
||||
from src.api.filters import FilterError, build_where, parse_aggregators
|
||||
from src.db import aggregate_marches, count_marches, query_marches
|
||||
from src.db import schema as duckdb_schema
|
||||
from src.utils.data import DATA_SCHEMA
|
||||
|
||||
bp = Blueprint(
|
||||
"api_v1",
|
||||
"api_v1",
|
||||
url_prefix="/api/v1",
|
||||
description="API privée decp.info — accès tabulaire aux marchés publics.",
|
||||
)
|
||||
|
||||
MAX_PAGE_SIZE = 1000
|
||||
|
||||
|
||||
def _parse_pagination():
|
||||
try:
|
||||
page = int(request.args.get("page", "1"))
|
||||
page_size = int(request.args.get("page_size", "50"))
|
||||
except ValueError:
|
||||
abort(400, message="page et page_size doivent être des entiers")
|
||||
if page < 1:
|
||||
abort(400, message="page doit être >= 1")
|
||||
if page_size < 1 or page_size > MAX_PAGE_SIZE:
|
||||
abort(
|
||||
400,
|
||||
message=f"page_size doit être dans [1, {MAX_PAGE_SIZE}]",
|
||||
)
|
||||
return page, page_size
|
||||
|
||||
|
||||
def _parse_columns():
|
||||
raw = request.args.get("columns")
|
||||
if not raw:
|
||||
return None
|
||||
cols = [c.strip() for c in raw.split(",") if c.strip()]
|
||||
unknown = [c for c in cols if c not in duckdb_schema]
|
||||
if unknown:
|
||||
abort(400, message=f"Colonnes inconnues : {unknown}")
|
||||
return cols
|
||||
|
||||
|
||||
def _build_links(page, page_size, total):
|
||||
base = request.path
|
||||
qs = request.args.to_dict(flat=False)
|
||||
qs.pop("page", None)
|
||||
|
||||
def url_for(p):
|
||||
from urllib.parse import urlencode
|
||||
|
||||
params = [(k, v) for k, vs in qs.items() for v in vs]
|
||||
params.append(("page", str(p)))
|
||||
return f"{base}?{urlencode(params)}"
|
||||
|
||||
prev_url = url_for(page - 1) if page > 1 else None
|
||||
next_url = None
|
||||
if total is None or page * page_size < total:
|
||||
next_url = url_for(page + 1)
|
||||
return {"prev": prev_url, "next": next_url}
|
||||
|
||||
|
||||
@bp.after_request
|
||||
def _track_consumption(response):
|
||||
token_id = getattr(g, "token_id", None)
|
||||
if token_id is not None:
|
||||
tracking.enqueue_counter_update(token_id)
|
||||
tracking.enqueue_matomo_event(
|
||||
token_id=token_id,
|
||||
path=request.path,
|
||||
query_string=request.query_string.decode("utf-8", errors="replace"),
|
||||
status_code=response.status_code,
|
||||
user_agent=request.headers.get("User-Agent", ""),
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@bp.route("/health")
|
||||
def health():
|
||||
"""Sonde de santé, sans authentification."""
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@bp.route("/schema")
|
||||
def schema():
|
||||
"""Liste des champs disponibles dans le dataset DECP (format TableSchema)."""
|
||||
return {"fields": list(DATA_SCHEMA.values())}
|
||||
|
||||
|
||||
@bp.route("/data")
|
||||
@bp.doc(
|
||||
security=[{"BearerAuth": []}],
|
||||
parameters=[
|
||||
{
|
||||
"name": "page",
|
||||
"in": "query",
|
||||
"schema": {"type": "integer", "default": 1, "minimum": 1},
|
||||
"description": "Numéro de page (commence à 1).",
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"in": "query",
|
||||
"schema": {"type": "integer", "default": 50, "minimum": 1, "maximum": 1000},
|
||||
"description": "Nombre de résultats par page (max 1000).",
|
||||
},
|
||||
{
|
||||
"name": "columns",
|
||||
"in": "query",
|
||||
"schema": {"type": "string"},
|
||||
"description": "Liste de colonnes à retourner, séparées par des virgules (ex: `id,acheteur_id,montant`). Par défaut : toutes.",
|
||||
},
|
||||
{
|
||||
"name": "count_results",
|
||||
"in": "query",
|
||||
"schema": {"type": "string", "enum": ["true", "false"], "default": "true"},
|
||||
"description": "Inclure le total (`COUNT(*)`) dans `meta`. Mettre `false` pour accélérer la requête. Ignoré en mode agrégation.",
|
||||
},
|
||||
{
|
||||
"name": "<colonne>__<opérateur>",
|
||||
"in": "query",
|
||||
"schema": {"type": "string"},
|
||||
"description": (
|
||||
"Filtre ou agrégation dynamique : `<colonne>__<opérateur>` "
|
||||
"(voir les colonnes via `/schema`).\n\n"
|
||||
"**Filtres** (`<colonne>__<op>=<valeur>`) :\n"
|
||||
"- `exact` : égal à la valeur\n"
|
||||
"- `differs` : différent de la valeur (null-safe, `IS DISTINCT FROM`)\n"
|
||||
"- `contains` / `notcontains` : contient / ne contient pas (LIKE)\n"
|
||||
"- `in` / `notin` : dans / hors d'une liste séparée par des virgules\n"
|
||||
"- `less` / `greater` : ≤ / ≥\n"
|
||||
"- `strictly_less` / `strictly_greater` : < / >\n"
|
||||
"- `isnull` / `isnotnull` : valeur nulle / non nulle (sans valeur)\n"
|
||||
"- `sort` : tri, valeur `asc` ou `desc`\n\n"
|
||||
"**Agrégation** (drapeaux sans valeur, ex. `acheteur_departement_code__groupby&montant__sum`) :\n"
|
||||
"- `groupby` : regroupe sur la colonne\n"
|
||||
"- `count`, `sum`, `avg`, `min`, `max` : agrège la colonne ; "
|
||||
"la colonne de sortie est nommée `colonne__count`, `colonne__sum`, "
|
||||
"`colonne__avg`, `colonne__min`, `colonne__max`\n\n"
|
||||
"En mode agrégation, la réponse contient des lignes groupées, "
|
||||
"`columns` est interdit et `meta` ne contient pas `total`. "
|
||||
"`sort` peut être appliqué sur une colonne `groupby` (ex. `acheteur_departement_code__sort=asc`) ; "
|
||||
"il n'est pas supporté sur les alias d'agrégats (ex. `uid__count__sort=desc` → 400).\n\n"
|
||||
"Exemples : `acheteur_id__contains=VILLE`, `montant__greater=10000`, "
|
||||
"`acheteur_departement_code__groupby&montant__sum`, "
|
||||
"`acheteur_departement_code__groupby&uid__count&acheteur_departement_code__sort=asc`."
|
||||
),
|
||||
},
|
||||
],
|
||||
)
|
||||
@require_token
|
||||
def data():
|
||||
"""Récupère des marchés publics filtrés, triés ou agrégés.
|
||||
|
||||
Filtres en query string : `<colonne>__<opérateur>=<valeur>`.
|
||||
Opérateurs de filtre : exact, differs, contains, notcontains, in, notin,
|
||||
less, greater, strictly_less, strictly_greater, isnull, isnotnull, sort.
|
||||
|
||||
Agrégation (drapeaux sans valeur) : `<colonne>__groupby`,
|
||||
`<colonne>__count|sum|avg|min|max`. Les colonnes agrégées sont nommées
|
||||
`<colonne>__<opérateur>`. `columns` est interdit avec une agrégation et
|
||||
`meta` ne contient alors pas `total`. `sort` est supporté sur les colonnes
|
||||
`groupby` ; non supporté sur les alias d'agrégats (→ 400).
|
||||
|
||||
Paramètres réservés : page (défaut 1), page_size (défaut 50, max 1000),
|
||||
columns (csv), count_results (true|false ; mettre false pour économiser
|
||||
le COUNT(*)).
|
||||
|
||||
Exemple d'agrégation :
|
||||
`?acheteur_departement_code__groupby&uid__count&montant__sum`
|
||||
"""
|
||||
import polars as pl
|
||||
import polars.selectors as cs
|
||||
|
||||
page, page_size = _parse_pagination()
|
||||
columns = _parse_columns()
|
||||
count_results = request.args.get("count_results", "true").lower() != "false"
|
||||
|
||||
args = list(request.args.items(multi=True))
|
||||
try:
|
||||
agg = parse_aggregators(args, duckdb_schema)
|
||||
where_sql, params, order_sql = build_where(args, duckdb_schema)
|
||||
except FilterError as e:
|
||||
abort(400, message=str(e), errors={"field": e.field})
|
||||
|
||||
if agg is not None:
|
||||
if columns:
|
||||
abort(
|
||||
400,
|
||||
message="`columns` ne peut pas être combiné avec une agrégation",
|
||||
)
|
||||
df = aggregate_marches(
|
||||
select_sql=agg.select_sql,
|
||||
where_sql=where_sql,
|
||||
params=params,
|
||||
group_by=agg.group_by_sql,
|
||||
order_by=order_sql or None,
|
||||
limit=page_size,
|
||||
offset=(page - 1) * page_size,
|
||||
)
|
||||
df_ready = df.with_columns(cs.temporal().cast(pl.String))
|
||||
# Si la page est partielle, on connaît le total exact ; sinon on ne sait pas.
|
||||
agg_total = (
|
||||
(page - 1) * page_size + df.height if df.height < page_size else None
|
||||
)
|
||||
return {
|
||||
"data": df_ready.to_dicts(),
|
||||
"meta": {"page": page, "page_size": page_size},
|
||||
"links": _build_links(page, page_size, agg_total),
|
||||
}
|
||||
|
||||
df = query_marches(
|
||||
where_sql=where_sql,
|
||||
params=params,
|
||||
columns=columns,
|
||||
order_by=order_sql,
|
||||
limit=page_size,
|
||||
offset=(page - 1) * page_size,
|
||||
)
|
||||
|
||||
# JSON ne sérialise pas date/datetime nativement → cast en string ISO
|
||||
df_ready = df.with_columns(cs.temporal().cast(pl.String))
|
||||
|
||||
total = count_marches(where_sql, params) if count_results else None
|
||||
meta = {"page": page, "page_size": page_size}
|
||||
if total is not None:
|
||||
meta["total"] = total
|
||||
|
||||
return {
|
||||
"data": df_ready.to_dicts(),
|
||||
"meta": meta,
|
||||
"links": _build_links(page, page_size, total),
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
from src.api import tokens_db
|
||||
|
||||
|
||||
def main(argv=None, env=None) -> int:
|
||||
env = env if env is not None else os.environ
|
||||
parser = argparse.ArgumentParser(prog="python -m src.api.tokens_cli")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
p_create = sub.add_parser("create", help="Créer un token API")
|
||||
p_create.add_argument("--label", required=True)
|
||||
p_create.add_argument("--user-id", type=int, default=None)
|
||||
|
||||
sub.add_parser("list", help="Lister les tokens")
|
||||
|
||||
p_revoke = sub.add_parser("revoke", help="Révoquer un token")
|
||||
p_revoke.add_argument("token_id", type=int)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
db_path = env["USERS_DB_PATH"]
|
||||
tokens_db.init_schema(db_path)
|
||||
|
||||
if args.cmd == "create":
|
||||
token, token_id = tokens_db.create_token(db_path, args.label, args.user_id)
|
||||
print(f"id={token_id} label={args.label}")
|
||||
print(f"token (à conserver, ne sera plus affiché) : {token}")
|
||||
return 0
|
||||
|
||||
if args.cmd == "list":
|
||||
rows = tokens_db.list_tokens(db_path)
|
||||
if not rows:
|
||||
print("(aucun token)")
|
||||
return 0
|
||||
print(
|
||||
f"{'id':<4} {'label':<40} {'created_at':<26} {'last_used_at':<26} {'count':<7} revoked"
|
||||
)
|
||||
for r in rows:
|
||||
print(
|
||||
f"{r['id']:<4} {r['label']:<40} {r['created_at']:<26} "
|
||||
f"{(r['last_used_at'] or '-'):<26} {r['count_total']:<7} "
|
||||
f"{r['revoked_at'] or ''}"
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.cmd == "revoke":
|
||||
tokens_db.revoke_token(db_path, args.token_id)
|
||||
print(f"token id={args.token_id} révoqué")
|
||||
return 0
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,94 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
TOKEN_PREFIX = "decpinfo_"
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS api_tokens (
|
||||
id INTEGER PRIMARY KEY,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
label TEXT NOT NULL,
|
||||
user_id INTEGER,
|
||||
created_at TEXT NOT NULL,
|
||||
last_used_at TEXT,
|
||||
count_total INTEGER NOT NULL DEFAULT 0,
|
||||
revoked_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_api_tokens_hash ON api_tokens(token_hash);
|
||||
"""
|
||||
|
||||
|
||||
def _utcnow_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _hash(token: str) -> str:
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _connect(db_path):
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def init_schema(db_path) -> None:
|
||||
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with _connect(db_path) as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def create_token(db_path, label: str, user_id: int | None = None) -> tuple[str, int]:
|
||||
token = TOKEN_PREFIX + secrets.token_hex(32)
|
||||
with _connect(db_path) as conn:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO api_tokens (token_hash, label, user_id, created_at) "
|
||||
"VALUES (?, ?, ?, ?)",
|
||||
(_hash(token), label, user_id, _utcnow_iso()),
|
||||
)
|
||||
conn.commit()
|
||||
return token, cur.lastrowid
|
||||
|
||||
|
||||
def get_token_by_plaintext(db_path, token: str) -> dict | None:
|
||||
with _connect(db_path) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM api_tokens WHERE token_hash = ?",
|
||||
(_hash(token),),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def revoke_token(db_path, token_id: int) -> None:
|
||||
with _connect(db_path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE api_tokens SET revoked_at = ? WHERE id = ?",
|
||||
(_utcnow_iso(), token_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def increment_usage(db_path, token_id: int) -> None:
|
||||
with _connect(db_path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE api_tokens "
|
||||
"SET count_total = count_total + 1, last_used_at = ? "
|
||||
"WHERE id = ?",
|
||||
(_utcnow_iso(), token_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def list_tokens(db_path) -> list[dict]:
|
||||
with _connect(db_path) as conn:
|
||||
rows = conn.execute("SELECT * FROM api_tokens ORDER BY id").fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
@@ -0,0 +1,110 @@
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from src.api import tokens_db
|
||||
from src.utils import logger
|
||||
|
||||
_STOP_SENTINEL = object()
|
||||
_queue: Optional[queue.Queue] = None
|
||||
_worker_thread: Optional[threading.Thread] = None
|
||||
|
||||
|
||||
def _worker_loop(q: queue.Queue, db_path: str) -> None:
|
||||
while True:
|
||||
item = q.get()
|
||||
try:
|
||||
if item is _STOP_SENTINEL:
|
||||
return
|
||||
kind, payload = item
|
||||
if kind == "counter":
|
||||
token_id = payload
|
||||
try:
|
||||
tokens_db.increment_usage(db_path, token_id)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning(
|
||||
"tracking: échec increment_usage token_id=%s",
|
||||
token_id,
|
||||
exc_info=True,
|
||||
)
|
||||
elif kind == "matomo":
|
||||
try:
|
||||
_post_matomo(**payload)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("tracking: échec envoi Matomo", exc_info=True)
|
||||
finally:
|
||||
q.task_done()
|
||||
|
||||
|
||||
def _post_matomo(url: str, params: dict) -> None:
|
||||
"""POST fire-and-forget vers la Tracking API Matomo. Mockable en test."""
|
||||
httpx.post(url, data=params, timeout=5.0)
|
||||
|
||||
|
||||
def enqueue_matomo_event(
|
||||
token_id: int,
|
||||
path: str,
|
||||
query_string: str,
|
||||
status_code: int,
|
||||
user_agent: str,
|
||||
) -> None:
|
||||
if _queue is None:
|
||||
return
|
||||
if os.getenv("MATOMO_TRACKING_ENABLED", "false").lower() != "true":
|
||||
return
|
||||
url = os.getenv("MATOMO_URL")
|
||||
site_id = os.getenv("MATOMO_SITE_ID")
|
||||
if not url or not site_id:
|
||||
return
|
||||
full_url = f"https://decp.info{path}"
|
||||
if query_string:
|
||||
full_url += f"?{query_string}"
|
||||
params = {
|
||||
"idsite": site_id,
|
||||
"rec": "1",
|
||||
"url": full_url,
|
||||
"action_name": f"API {path}",
|
||||
"uid": f"token-{token_id}",
|
||||
"dimension1": str(token_id),
|
||||
"dimension2": str(status_code),
|
||||
"ua": user_agent,
|
||||
}
|
||||
_queue.put(("matomo", {"url": url, "params": params}))
|
||||
|
||||
|
||||
def start_worker(db_path: str) -> None:
|
||||
global _queue, _worker_thread
|
||||
if _worker_thread is not None and _worker_thread.is_alive():
|
||||
return
|
||||
_queue = queue.Queue()
|
||||
_worker_thread = threading.Thread(
|
||||
target=_worker_loop, args=(_queue, db_path), daemon=True
|
||||
)
|
||||
_worker_thread.start()
|
||||
|
||||
|
||||
def stop_worker() -> None:
|
||||
global _worker_thread, _queue
|
||||
if _worker_thread is None:
|
||||
return
|
||||
_queue.put(_STOP_SENTINEL)
|
||||
_worker_thread.join(timeout=2.0)
|
||||
_worker_thread = None
|
||||
_queue = None
|
||||
|
||||
|
||||
def enqueue_counter_update(token_id: int) -> None:
|
||||
if _queue is None:
|
||||
return # tracking désactivé (tests par ex.)
|
||||
_queue.put(("counter", token_id))
|
||||
|
||||
|
||||
def flush(timeout: float = 2.0) -> None:
|
||||
"""Attend que la queue soit drainée. Utile en test."""
|
||||
q = _queue
|
||||
if q is None:
|
||||
return
|
||||
q.join()
|
||||
+27
-9
@@ -2,6 +2,7 @@ import os
|
||||
from shutil import rmtree
|
||||
|
||||
import dash_bootstrap_components as dbc
|
||||
import pandas # noqa: F401 # eager import: avoid plotly's lazy-import race across Dash callback threads
|
||||
import tomllib
|
||||
from dash import (
|
||||
Dash,
|
||||
@@ -15,7 +16,7 @@ from dash import (
|
||||
page_registry,
|
||||
)
|
||||
from dotenv import load_dotenv
|
||||
from flask import Response
|
||||
from flask import Flask, Response
|
||||
from flask_login import current_user
|
||||
|
||||
from src.auth.setup import init_auth
|
||||
@@ -38,12 +39,10 @@ META_TAGS = [
|
||||
if DEVELOPMENT:
|
||||
META_TAGS.append({"name": "robots", "content": "noindex"})
|
||||
|
||||
app: Dash = Dash(
|
||||
title="decp.info",
|
||||
use_pages=True,
|
||||
compress=True,
|
||||
meta_tags=META_TAGS,
|
||||
)
|
||||
# Le cache doit être initialisé AVANT la construction de Dash : `use_pages=True`
|
||||
# importe les modules de pages pendant l'instanciation, et certains appellent des
|
||||
# fonctions memoizées (@cache.memoize) dès l'import (ex. tableau.py).
|
||||
server = Flask(__name__)
|
||||
|
||||
cache_dir = os.getenv("CACHE_DIR", "/tmp/decp-cache")
|
||||
|
||||
@@ -51,7 +50,7 @@ if os.path.exists(cache_dir):
|
||||
rmtree(cache_dir)
|
||||
|
||||
cache.init_app(
|
||||
app.server,
|
||||
server,
|
||||
config={
|
||||
"CACHE_TYPE": "FileSystemCache",
|
||||
"CACHE_DIR": cache_dir,
|
||||
@@ -62,6 +61,14 @@ cache.init_app(
|
||||
},
|
||||
)
|
||||
|
||||
app: Dash = Dash(
|
||||
server=server,
|
||||
title="decp.info",
|
||||
use_pages=True,
|
||||
compress=True,
|
||||
meta_tags=META_TAGS,
|
||||
)
|
||||
|
||||
init_auth(app.server)
|
||||
|
||||
# Exempter les routes internes de Dash de la protection CSRF.
|
||||
@@ -76,6 +83,10 @@ if _auth_csrf is not None:
|
||||
if _vf is not None:
|
||||
_auth_csrf.exempt(_vf)
|
||||
|
||||
from src.api import init_api # noqa: E402 # inline: src.db.conn must be ready first
|
||||
|
||||
init_api(app.server)
|
||||
|
||||
|
||||
# robots.txt
|
||||
@app.server.route("/robots.txt")
|
||||
@@ -94,6 +105,7 @@ def sitemap():
|
||||
"/observatoire",
|
||||
"/tableau",
|
||||
"/a-propos",
|
||||
"/etapes",
|
||||
]
|
||||
xml = '<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
xml += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
|
||||
@@ -170,7 +182,13 @@ navbar = dbc.Navbar(
|
||||
style={"minWidth": "230px"},
|
||||
),
|
||||
dbc.Nav(
|
||||
children=[dcc.Markdown(os.getenv("ANNOUNCEMENTS"), id="announcements")],
|
||||
children=[
|
||||
dcc.Markdown(
|
||||
os.getenv("ANNOUNCEMENTS"),
|
||||
id="announcements",
|
||||
dangerously_allow_html=True,
|
||||
),
|
||||
],
|
||||
style={
|
||||
"maxWidth": "1200px",
|
||||
"display": "inline-block",
|
||||
|
||||
@@ -144,6 +144,10 @@ p.version > a {
|
||||
max-width: 900px;
|
||||
}
|
||||
|
||||
#announcements p {
|
||||
margin-bottom: 0.2rem;
|
||||
}
|
||||
|
||||
.seeBorder {
|
||||
border: dotted 1px green;
|
||||
}
|
||||
@@ -173,6 +177,10 @@ p.version > a {
|
||||
margin-right: 12px;
|
||||
}
|
||||
|
||||
#search_results td[data-dash-column="Département"] {
|
||||
text-wrap: wrap;
|
||||
}
|
||||
|
||||
/* --- Dashboard inputs --- */
|
||||
|
||||
.Select--multi .Select-value {
|
||||
@@ -351,10 +359,68 @@ table.cell-table th {
|
||||
right: 200px;
|
||||
}
|
||||
|
||||
/* ===== Tableaux : en-têtes collants + scroll horizontal (#82) ===== */
|
||||
|
||||
/* Contenir le scroll horizontal dans le conteneur Dash (élimine la scrollbar native en bas de page) */
|
||||
/* overflow-y:clip évite la conversion CSS visible→auto qui ajouterait une scrollbar verticale */
|
||||
.marches_table .dash-spreadsheet-container {
|
||||
overflow-x: hidden !important;
|
||||
overflow-y: clip !important;
|
||||
}
|
||||
|
||||
/* L'inner reste visible pour que le tableau se déploie librement en largeur */
|
||||
.marches_table .dash-spreadsheet-inner {
|
||||
overflow: visible !important;
|
||||
}
|
||||
|
||||
.marches_table .cell-table tr:nth-child(even) td {
|
||||
background-color: rgb(255 240 240 / 40%);
|
||||
}
|
||||
|
||||
/* Colonne "Marché" : lien loupe centré et sans soulignement */
|
||||
table.cell-table td[data-dash-column="marche"] .dash-cell-value {
|
||||
text-align: center !important;
|
||||
}
|
||||
|
||||
td[data-dash-column="marche"] a {
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
th[data-dash-column="marche"].dash-filter input {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Barre de défilement horizontale, collée en haut du tableau */
|
||||
.marches_table .dt-hscroll {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 11;
|
||||
height: 12px;
|
||||
background-color: #e0e0e0;
|
||||
overflow: hidden;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.marches_table .dt-hscroll.is-hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Thumb custom — toujours visible, draggable */
|
||||
.marches_table .dt-hscroll-thumb {
|
||||
position: absolute;
|
||||
top: 2px;
|
||||
height: calc(100% - 4px);
|
||||
min-width: 40px;
|
||||
background-color: orange;
|
||||
border-radius: 8px;
|
||||
cursor: grab;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.marches_table .dt-hscroll-thumb:active {
|
||||
cursor: grabbing;
|
||||
}
|
||||
|
||||
/* Column Visibility Menu */
|
||||
.column-actions {
|
||||
margin-right: 8px;
|
||||
@@ -542,3 +608,218 @@ input[type="number"]::-webkit-inner-spin-button {
|
||||
input[type="number"] {
|
||||
-moz-appearance: textfield;
|
||||
}
|
||||
|
||||
/* ===== Page /etapes : graphique données par étape et par seuil ===== */
|
||||
|
||||
.etapes-chart-scroll {
|
||||
overflow-x: auto;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.etapes-chart {
|
||||
min-width: 720px;
|
||||
background: #fff;
|
||||
border: 1px solid #d0d5dd;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
font-size: 13px;
|
||||
display: grid;
|
||||
grid-template-columns: 150px repeat(5, 1fr);
|
||||
}
|
||||
|
||||
.etapes-corner {
|
||||
border-bottom: 2px solid #344054;
|
||||
}
|
||||
|
||||
.etapes-xhead {
|
||||
grid-column: 2 / -1;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, 1fr);
|
||||
border-bottom: 2px solid #344054;
|
||||
}
|
||||
|
||||
.etapes-xcell {
|
||||
text-align: center;
|
||||
padding: 6px 2px;
|
||||
font-size: 11px;
|
||||
color: #475467;
|
||||
border-left: 1px dashed #d0d5dd;
|
||||
}
|
||||
|
||||
.etapes-xcell strong {
|
||||
display: block;
|
||||
color: #101828;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.etapes-stage {
|
||||
padding: 14px 10px;
|
||||
font-weight: 600;
|
||||
color: #101828;
|
||||
border-bottom: 1px solid #eaecf0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.etapes-stage small {
|
||||
font-weight: 400;
|
||||
color: #667085;
|
||||
}
|
||||
|
||||
.etapes-lane {
|
||||
grid-column: 2 / -1;
|
||||
position: relative;
|
||||
border-bottom: 1px solid #eaecf0;
|
||||
min-height: 52px;
|
||||
}
|
||||
|
||||
.etapes-segs {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, 1fr);
|
||||
}
|
||||
|
||||
.etapes-segs > div {
|
||||
border-left: 1px dashed #eaecf0;
|
||||
}
|
||||
|
||||
.etapes-bar {
|
||||
position: absolute;
|
||||
top: 9px;
|
||||
height: 32px;
|
||||
border-radius: 6px;
|
||||
color: #fff;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 0 10px;
|
||||
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.12);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
cursor: pointer;
|
||||
transition: filter 0.15s, box-shadow 0.15s;
|
||||
}
|
||||
|
||||
.etapes-bar:hover {
|
||||
filter: brightness(1.12);
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.22);
|
||||
}
|
||||
|
||||
.etapes-empty {
|
||||
color: #98a2b3;
|
||||
font-style: italic;
|
||||
padding: 14px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.etapes-note {
|
||||
margin-top: 8px;
|
||||
color: #667085;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* --- Vue mobile (liste par étape) : masquée par défaut --- */
|
||||
|
||||
.etapes-mobile {
|
||||
display: none;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.etapes-m-block {
|
||||
border: 1px solid #d0d5dd;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 12px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.etapes-m-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
background: #f9fafb;
|
||||
border-bottom: 1px solid #eaecf0;
|
||||
}
|
||||
|
||||
.etapes-m-stage {
|
||||
margin: 0;
|
||||
padding: 10px 12px;
|
||||
font-size: 15px;
|
||||
color: #101828;
|
||||
}
|
||||
|
||||
.etapes-m-link {
|
||||
background: none;
|
||||
border: none;
|
||||
color: #1570ef;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
padding: 0 12px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.etapes-m-link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
.etapes-m-item {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 8px;
|
||||
padding: 8px 12px;
|
||||
border-bottom: 1px solid #f2f4f7;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.etapes-m-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.etapes-m-item i {
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border-radius: 3px;
|
||||
flex: 0 0 auto;
|
||||
position: relative;
|
||||
top: 2px;
|
||||
}
|
||||
|
||||
.etapes-m-label {
|
||||
font-weight: 600;
|
||||
color: #101828;
|
||||
}
|
||||
|
||||
.etapes-m-seuil {
|
||||
color: #667085;
|
||||
}
|
||||
|
||||
.etapes-m-empty {
|
||||
color: #98a2b3;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.etapes-detail {
|
||||
margin: 1rem 0;
|
||||
padding: 1rem 1.25rem;
|
||||
border: 1px solid #d0d5dd;
|
||||
border-radius: 8px;
|
||||
background: #f9fafb;
|
||||
}
|
||||
|
||||
.etapes-detail:empty {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* --- Bascule desktop / mobile au point de rupture 768 px --- */
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.etapes-chart-scroll {
|
||||
display: none;
|
||||
}
|
||||
.etapes-mobile {
|
||||
display: block;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
// Barre de défilement horizontale pour les tableaux (.marches_table) — #82
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
function setup(wrapper) {
|
||||
if (wrapper.dataset.hscrollReady === "1") return;
|
||||
|
||||
const dashContainer = wrapper.querySelector(".dash-spreadsheet-container");
|
||||
if (!dashContainer) return;
|
||||
|
||||
// Garde posée après la vérification de dashContainer, avant toute manipulation DOM
|
||||
// qui déclencherait rootObs et provoquerait une re-entrée dans setup().
|
||||
wrapper.dataset.hscrollReady = "1";
|
||||
|
||||
const bar = document.createElement("div");
|
||||
bar.className = "dt-hscroll is-hidden";
|
||||
const thumb = document.createElement("div");
|
||||
thumb.className = "dt-hscroll-thumb";
|
||||
bar.appendChild(thumb);
|
||||
wrapper.insertBefore(bar, wrapper.firstChild);
|
||||
|
||||
// Métriques basées sur le conteneur scrollable (pas la page).
|
||||
// dashContainer a overflow-x:hidden → scrollLeft est contrôlable par JS.
|
||||
const metrics = () => {
|
||||
const total = dashContainer.scrollWidth;
|
||||
const visible = dashContainer.clientWidth;
|
||||
const thumbW = Math.max(40, (visible / total) * visible);
|
||||
const scrollRange = total - visible;
|
||||
const thumbRange = visible - thumbW;
|
||||
return { total, visible, thumbW, scrollRange, thumbRange };
|
||||
};
|
||||
|
||||
// Mise à jour de la position du thumb selon dashContainer.scrollLeft.
|
||||
const syncThumb = () => {
|
||||
const { total, visible, thumbW, scrollRange, thumbRange } = metrics();
|
||||
if (total <= visible + 1 || thumbRange <= 0) return;
|
||||
thumb.style.width = thumbW + "px";
|
||||
const fraction =
|
||||
scrollRange > 0 ? dashContainer.scrollLeft / scrollRange : 0;
|
||||
thumb.style.left = Math.round(fraction * thumbRange) + "px";
|
||||
};
|
||||
|
||||
// --- Drag souris + tactile ---
|
||||
let dragStartX = null;
|
||||
let dragScrollStart = null;
|
||||
|
||||
const startDrag = (clientX) => {
|
||||
dragStartX = clientX;
|
||||
dragScrollStart = dashContainer.scrollLeft;
|
||||
};
|
||||
const moveDrag = (clientX) => {
|
||||
if (dragStartX === null) return;
|
||||
const dx = clientX - dragStartX;
|
||||
const { scrollRange, thumbRange } = metrics();
|
||||
if (thumbRange <= 0) return;
|
||||
dashContainer.scrollLeft = Math.max(
|
||||
0,
|
||||
Math.min(scrollRange, dragScrollStart + (dx / thumbRange) * scrollRange)
|
||||
);
|
||||
};
|
||||
const endDrag = () => {
|
||||
dragStartX = null;
|
||||
};
|
||||
|
||||
thumb.addEventListener("mousedown", (e) => {
|
||||
startDrag(e.clientX);
|
||||
e.preventDefault();
|
||||
});
|
||||
document.addEventListener("mousemove", (e) => moveDrag(e.clientX));
|
||||
document.addEventListener("mouseup", endDrag);
|
||||
|
||||
thumb.addEventListener(
|
||||
"touchstart",
|
||||
(e) => {
|
||||
startDrag(e.touches[0].clientX);
|
||||
e.preventDefault();
|
||||
},
|
||||
{ passive: false }
|
||||
);
|
||||
document.addEventListener(
|
||||
"touchmove",
|
||||
(e) => {
|
||||
if (dragStartX !== null) {
|
||||
moveDrag(e.touches[0].clientX);
|
||||
e.preventDefault();
|
||||
}
|
||||
},
|
||||
{ passive: false }
|
||||
);
|
||||
document.addEventListener("touchend", endDrag);
|
||||
|
||||
// Clic sur le track (hors thumb) : saute à la position cliquée.
|
||||
bar.addEventListener("click", (e) => {
|
||||
if (e.target === thumb) return;
|
||||
const rect = bar.getBoundingClientRect();
|
||||
const { scrollRange, thumbW, thumbRange } = metrics();
|
||||
const fraction = Math.max(
|
||||
0,
|
||||
Math.min(1, (e.clientX - rect.left - thumbW / 2) / thumbRange)
|
||||
);
|
||||
dashContainer.scrollLeft = fraction * scrollRange;
|
||||
});
|
||||
|
||||
// Scroll molette/trackpad horizontal → redirigé vers le conteneur.
|
||||
// SPA : ce listener est intentionnellement conservé pour toute la durée de vie de la page.
|
||||
wrapper.addEventListener(
|
||||
"wheel",
|
||||
(e) => {
|
||||
if (Math.abs(e.deltaX) <= Math.abs(e.deltaY)) return;
|
||||
e.preventDefault();
|
||||
dashContainer.scrollLeft = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
dashContainer.scrollWidth - dashContainer.clientWidth,
|
||||
dashContainer.scrollLeft + e.deltaX
|
||||
)
|
||||
);
|
||||
},
|
||||
{ passive: false }
|
||||
);
|
||||
|
||||
// Synchronise le thumb quand le conteneur défile (drag, wheel, ou autre).
|
||||
// SPA : ce listener est intentionnellement conservé pour toute la durée de vie de la page.
|
||||
dashContainer.addEventListener("scroll", syncThumb);
|
||||
|
||||
const refresh = () => {
|
||||
const hasOverflow =
|
||||
dashContainer.scrollWidth > dashContainer.clientWidth + 1;
|
||||
bar.classList.toggle("is-hidden", !hasOverflow);
|
||||
if (hasOverflow) syncThumb();
|
||||
};
|
||||
|
||||
// Recalcule quand le tableau change (pagination, tri, filtre, données).
|
||||
const obs = new MutationObserver(() => refresh());
|
||||
obs.observe(dashContainer, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
});
|
||||
// SPA : ce listener est intentionnellement conservé pour toute la durée de vie de la page.
|
||||
window.addEventListener("resize", refresh);
|
||||
|
||||
refresh();
|
||||
}
|
||||
|
||||
function scan() {
|
||||
document.querySelectorAll(".marches_table").forEach(setup);
|
||||
}
|
||||
|
||||
// Les tableaux apparaissent après le rendu Dash : observer le body.
|
||||
const rootObs = new MutationObserver(() => scan());
|
||||
rootObs.observe(document.body, { childList: true, subtree: true });
|
||||
scan();
|
||||
})();
|
||||
@@ -8,27 +8,31 @@ import polars as pl
|
||||
import polars.selectors as cs
|
||||
from polars.exceptions import ComputeError
|
||||
|
||||
from src.utils import logger
|
||||
from src.utils import get_last_modified, logger
|
||||
|
||||
|
||||
def should_rebuild(db_path: Path, parquet_path: Path) -> bool:
|
||||
def should_rebuild(db_path: Path, parquet_path: str) -> bool:
|
||||
db_path = Path(db_path)
|
||||
parquet_path = Path(parquet_path)
|
||||
if not db_path.exists():
|
||||
return True
|
||||
dev = os.getenv("DEVELOPMENT", "False").lower() == "true"
|
||||
force = os.getenv("REBUILD_DUCKDB", "False").lower() == "true"
|
||||
if dev and not force:
|
||||
return False
|
||||
return parquet_path.stat().st_mtime > db_path.stat().st_mtime
|
||||
last_modified: float = get_last_modified(parquet_path)
|
||||
return last_modified > db_path.stat().st_mtime
|
||||
|
||||
|
||||
def _load_source_frame(parquet_path: Path) -> pl.DataFrame:
|
||||
def _load_source_frame() -> pl.DataFrame:
|
||||
"""Read the source parquet and apply the row-level transforms.
|
||||
|
||||
Kept here (not in utils.py) so src.db has no dependency on utils.
|
||||
Mirrors the behavior previously in utils.get_decp_data().
|
||||
"""
|
||||
|
||||
parquet_path: str = os.getenv("DATA_FILE_PARQUET_PATH", "")
|
||||
if not (parquet_path.startswith("http")):
|
||||
assert os.path.exists(parquet_path)
|
||||
try:
|
||||
lff: pl.LazyFrame = pl.scan_parquet(str(parquet_path))
|
||||
except ComputeError:
|
||||
@@ -58,20 +62,21 @@ def _load_source_frame(parquet_path: Path) -> pl.DataFrame:
|
||||
return lff.collect()
|
||||
|
||||
|
||||
def build_database(db_path: Path, parquet_path: Path) -> None:
|
||||
def build_database(db_path: Path) -> None:
|
||||
"""Build the DuckDB database atomically under an exclusive lock.
|
||||
|
||||
Caller MUST hold the fcntl.flock on the .lock file.
|
||||
"""
|
||||
db_path = Path(db_path)
|
||||
parquet_path = Path(parquet_path)
|
||||
tmp_path = db_path.with_suffix(".duckdb.tmp")
|
||||
staging_parquet = db_path.with_suffix(".staging.parquet")
|
||||
if tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
|
||||
logger.info(f"Construction de la base DuckDB à partir de {parquet_path}...")
|
||||
frame = _load_source_frame(parquet_path)
|
||||
logger.info(
|
||||
f"Construction de la base DuckDB à partir de {os.getenv('DATA_FILE_PARQUET_PATH', '')}..."
|
||||
)
|
||||
frame = _load_source_frame()
|
||||
|
||||
# Write transformed frame as parquet so DuckDB can read it natively
|
||||
# (avoids pyarrow dependency for the Polars→DuckDB handoff)
|
||||
@@ -110,16 +115,27 @@ def build_database(db_path: Path, parquet_path: Path) -> None:
|
||||
|
||||
|
||||
def _ensure_database() -> Path:
|
||||
db_path = Path("./decp.duckdb")
|
||||
parquet_path = Path(os.getenv("DATA_FILE_PARQUET_PATH"))
|
||||
db_path = Path(os.getenv("DUCKDB_PATH", "./decp.duckdb"))
|
||||
parquet_path = os.getenv("DATA_FILE_PARQUET_PATH", "")
|
||||
lock_path = db_path.with_suffix(".duckdb.lock")
|
||||
db_exists = db_path.exists()
|
||||
|
||||
with open(lock_path, "w") as lock_fd:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
if should_rebuild(db_path, parquet_path):
|
||||
build_database(db_path, parquet_path)
|
||||
else:
|
||||
logger.debug("Base de données déjà disponible et à jour.")
|
||||
try:
|
||||
if should_rebuild(db_path, parquet_path):
|
||||
build_database(db_path)
|
||||
else:
|
||||
logger.debug("Base de données déjà disponible et à jour.")
|
||||
except Exception as e:
|
||||
if db_exists and db_path.exists():
|
||||
logger.error(
|
||||
f"Bootstrap données KO ({e}). "
|
||||
f"Réutilisation du DuckDB existant : {db_path}"
|
||||
)
|
||||
else:
|
||||
logger.critical("Aucune base DuckDB et reconstruction impossible.")
|
||||
raise
|
||||
return db_path
|
||||
|
||||
|
||||
@@ -135,10 +151,11 @@ def get_cursor() -> duckdb.DuckDBPyConnection:
|
||||
|
||||
def query_marches(
|
||||
where_sql: str = "TRUE",
|
||||
params: tuple = (),
|
||||
params: tuple | list = (),
|
||||
columns: list[str] | None = None,
|
||||
order_by: str | None = None,
|
||||
limit: int | None = None,
|
||||
offset: int | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""Run a parameterized SELECT against the decp table and return Polars.
|
||||
|
||||
@@ -152,4 +169,55 @@ def query_marches(
|
||||
sql += f" ORDER BY {order_by}"
|
||||
if limit is not None:
|
||||
sql += f" LIMIT {int(limit)}"
|
||||
if offset is not None:
|
||||
sql += f" OFFSET {int(offset)}"
|
||||
|
||||
logger.debug("query_marches: " + sql.replace("?", "{}").format(*params))
|
||||
|
||||
return get_cursor().execute(sql, list(params)).pl()
|
||||
|
||||
|
||||
def count_marches(where_sql: str = "TRUE", params: tuple | list = ()) -> int:
|
||||
"""Retourne le nombre de lignes correspondant à where_sql."""
|
||||
sql = f"SELECT COUNT(*) FROM decp WHERE {where_sql}"
|
||||
logger.debug("count_marches: " + sql.replace("?", "{}").format(*params))
|
||||
result = get_cursor().execute(sql, list(params)).fetchone()
|
||||
return int(result[0]) if result else 0
|
||||
|
||||
|
||||
def count_unique_marches(where_sql: str = "TRUE", params: tuple | list = ()) -> int:
|
||||
"""Retourne le nombre de uid distincts correspondant à where_sql."""
|
||||
sql = f"SELECT COUNT(DISTINCT uid) FROM decp WHERE {where_sql}"
|
||||
logger.debug("count_unique_marches: " + sql.replace("?", "{}").format(*params))
|
||||
result = get_cursor().execute(sql, list(params)).fetchone()
|
||||
return int(result[0]) if result else 0
|
||||
|
||||
|
||||
def aggregate_marches(
|
||||
select_sql: str,
|
||||
where_sql: str = "TRUE",
|
||||
params: tuple | list = (),
|
||||
group_by: str | None = None,
|
||||
order_by: str | None = None,
|
||||
limit: int | None = None,
|
||||
offset: int | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""SELECT agrégé paramétré contre la table decp.
|
||||
|
||||
`select_sql`, `group_by` et `order_by` sont des fragments SQL construits
|
||||
depuis des noms de colonnes validés (jamais de valeur utilisateur libre).
|
||||
Les valeurs de filtre passent par le binding `?` via `params`.
|
||||
"""
|
||||
sql = f"SELECT {select_sql} FROM decp WHERE {where_sql}"
|
||||
if group_by:
|
||||
sql += f" GROUP BY {group_by}"
|
||||
if order_by:
|
||||
sql += f" ORDER BY {order_by}"
|
||||
if limit is not None:
|
||||
sql += f" LIMIT {int(limit)}"
|
||||
if offset is not None:
|
||||
sql += f" OFFSET {int(offset)}"
|
||||
|
||||
logger.debug("aggregate_marches: " + sql.replace("?", "{}").format(*params))
|
||||
|
||||
return get_cursor().execute(sql, list(params)).pl()
|
||||
|
||||
+186
-27
@@ -1,6 +1,5 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
import dash_bootstrap_components as dbc
|
||||
import dash_leaflet as dl
|
||||
@@ -11,8 +10,10 @@ import plotly.graph_objects as go
|
||||
import polars as pl
|
||||
from dash import dash_table, dcc, html
|
||||
from dash_extensions.javascript import Namespace
|
||||
from polars.exceptions import ColumnNotFoundError
|
||||
|
||||
from src.db import schema
|
||||
from src.utils import logger
|
||||
from src.utils.data import DATA_SCHEMA, DEPARTEMENTS_GEOJSON
|
||||
from src.utils.table import add_links, format_number, setup_table_columns
|
||||
|
||||
@@ -118,9 +119,12 @@ def get_barchart_sources(lff: pl.LazyFrame, type_date: str):
|
||||
|
||||
def get_sources_tables(source_path) -> html.Div:
|
||||
try:
|
||||
if not source_path:
|
||||
raise ValueError("SOURCE_STATS_CSV_PATH non défini")
|
||||
dff = pl.read_csv(source_path)
|
||||
except (URLError, HTTPError):
|
||||
return html.Div("Erreur de connexion")
|
||||
except Exception as e:
|
||||
logger.warning(f"Sources de données indisponibles ({e})")
|
||||
return html.Div("Sources de données momentanément indisponibles.")
|
||||
dff = dff.with_columns(
|
||||
(
|
||||
pl.lit('<a href = "')
|
||||
@@ -173,39 +177,78 @@ def get_sources_tables(source_path) -> html.Div:
|
||||
return html.Div(children=datatable)
|
||||
|
||||
|
||||
def point_on_map(lat, lon):
|
||||
lat = float(lat)
|
||||
lon = float(lon)
|
||||
def point_on_map(lat, lon, departement_code=None):
|
||||
"""Fonction améliorée utilisant les codes départementaux pour la détection de région.
|
||||
|
||||
# Create a scatter mapbox or choropleth map
|
||||
Args:
|
||||
lat: Coordonnée de latitude
|
||||
lon: Coordonnée de longitude
|
||||
departement_code: Code du département (ex: '75', '971', etc.)
|
||||
|
||||
Returns:
|
||||
html.Div contenant la carte, ou div vide si invalide
|
||||
"""
|
||||
# Validation des coordonnées
|
||||
try:
|
||||
lat = float(lat)
|
||||
lon = float(lon)
|
||||
except (TypeError, ValueError):
|
||||
return html.Div() # Div vide pour les coordonnées invalides
|
||||
|
||||
# Vérification que les coordonnées sont valides
|
||||
if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
|
||||
return html.Div()
|
||||
|
||||
# Si aucun code département n'est fourni, retourner une div vide
|
||||
if not departement_code:
|
||||
return html.Div()
|
||||
|
||||
# Détermination de la région en utilisant le code département
|
||||
# Logique identique à get_geographic_maps
|
||||
if departement_code in ["971", "972", "973", "974", "976"]:
|
||||
region_key = departement_code # Département d'outre-mer
|
||||
elif len(departement_code) == 2: # Département métropolitain
|
||||
region_key = "Hexagone"
|
||||
else:
|
||||
return html.Div() # Format de code département invalide
|
||||
|
||||
# Paramètres de carte par région (réutilisés de get_geographic_maps)
|
||||
regions = {
|
||||
"Hexagone": {"center": [46.6, 2.2], "zoom": 5},
|
||||
"971": {"center": [16.23, -61.55], "zoom": 9}, # Guadeloupe
|
||||
"972": {"center": [14.64, -61.02], "zoom": 10}, # Martinique
|
||||
"973": {"center": [3.93, -53.12], "zoom": 7}, # Guyane
|
||||
"974": {"center": [-21.11, 55.53], "zoom": 9}, # La Réunion
|
||||
"976": {"center": [-12.82, 45.16], "zoom": 10}, # Mayotte
|
||||
}
|
||||
|
||||
settings = regions.get(region_key, regions["Hexagone"])
|
||||
|
||||
# Création de la carte
|
||||
fig = px.scatter_map(
|
||||
lat=[lat], lon=[lon], height=300, width=400, color=[1], size=[1]
|
||||
lat=[lat],
|
||||
lon=[lon],
|
||||
height=300,
|
||||
# width=400,
|
||||
color=[1],
|
||||
zoom=settings["zoom"],
|
||||
)
|
||||
|
||||
fig.update_coloraxes(showscale=False)
|
||||
fig.update_traces(marker=dict(size=10))
|
||||
|
||||
# Set map style (you can use 'open-street-map', 'carto-positron', etc.)
|
||||
# Configuration de la carte (interactive - zoomable)
|
||||
fig.update_layout(
|
||||
mapbox_style="light", # Light, clean background
|
||||
map_style="light", # Fond de carte clair
|
||||
margin={"r": 0, "t": 0, "l": 0, "b": 0},
|
||||
mapbox_center={"lat": settings["center"][0], "lon": settings["center"][1]},
|
||||
mapbox_zoom=settings["zoom"],
|
||||
coloraxis_showscale=False,
|
||||
)
|
||||
|
||||
# Optionally, center the map on France
|
||||
fig.update_geos(
|
||||
center=dict(lat=46.603354, lon=1.888334), # Center of France
|
||||
lataxis_range=[41, 51.5], # Latitude range for France
|
||||
lonaxis_range=[-5, 10], # Longitude range for France
|
||||
return html.Div(
|
||||
dcc.Graph(figure=fig, config={"displayModeBar": False}),
|
||||
)
|
||||
|
||||
# But scatter_mapbox doesn't use geos, so better to control via zoom/center manually
|
||||
# Let's reset and use proper centering in scatter_mapbox instead:
|
||||
|
||||
fig.update_layout(map_center={"lat": 46.6, "lon": 1.89}, map_zoom=4)
|
||||
|
||||
graph = dcc.Graph(id="map", figure=fig)
|
||||
graph = html.Div(style={"width": "400px"})
|
||||
return graph
|
||||
|
||||
|
||||
class DataTable(dash_table.DataTable):
|
||||
def __init__(
|
||||
@@ -685,6 +728,118 @@ def get_dashboard_summary_table(dff, dff_per_uid, nb_marches):
|
||||
return summary_table
|
||||
|
||||
|
||||
CONSIDERATIONS_REGEX = r"(?i)Clause|Critère|Marché réservé"
|
||||
CONSIDERATIONS_COLUMNS = {
|
||||
"sociales": "considerationsSociales",
|
||||
"environnementales": "considerationsEnvironnementales",
|
||||
}
|
||||
|
||||
|
||||
def compute_considerations_stats(lff: pl.LazyFrame) -> dict[str, tuple[int, int, int]]:
|
||||
"""Statistiques considérations pour l'observatoire.
|
||||
|
||||
Clés renvoyées :
|
||||
- "champs_renseignes" : (count_ren_sociales, pct / total)
|
||||
- "{key}_renseignees" : (count_ren, pct positifs parmi renseignés)
|
||||
Colonne absente -> (0, 0).
|
||||
"""
|
||||
names = lff.collect_schema().names()
|
||||
present = {key: col for key, col in CONSIDERATIONS_COLUMNS.items() if col in names}
|
||||
|
||||
stats: dict[str, tuple[int, int, int]] = {"champs_renseignes": (0, 0)}
|
||||
for key in CONSIDERATIONS_COLUMNS:
|
||||
stats[f"{key}_renseignees"] = (0, 0)
|
||||
|
||||
if not present:
|
||||
return stats
|
||||
|
||||
agg = (
|
||||
lff.select(["uid"] + list(present.values()))
|
||||
.group_by("uid")
|
||||
.agg([pl.col(col).first() for col in present.values()])
|
||||
.collect(engine="streaming")
|
||||
)
|
||||
|
||||
total = agg.height
|
||||
if total == 0:
|
||||
return stats
|
||||
|
||||
for key, col in present.items():
|
||||
count_ren = agg.filter(pl.col(col).is_not_null()).height
|
||||
count_pos_ren = agg.filter(
|
||||
pl.col(col).is_not_null() & (pl.col(col) != "Sans objet")
|
||||
).height
|
||||
pct_pos_ren = round(100 * count_pos_ren / count_ren) if count_ren > 0 else 0
|
||||
stats[f"{key}_renseignees"] = (count_ren, count_pos_ren, pct_pos_ren)
|
||||
if key == "sociales":
|
||||
stats["champs_renseignes"] = (count_ren, round(100 * count_ren / total))
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
# (clé stats, libellé, couleur)
|
||||
CONSIDERATIONS_RENSEIGNEES = [
|
||||
("sociales_renseignees", "Considérations sociales", "#CC6677"),
|
||||
("environnementales_renseignees", "Considérations environnementales", "#117733"),
|
||||
]
|
||||
|
||||
|
||||
def _progress_bar(pct: int, color: str) -> dbc.Progress:
|
||||
return dbc.Progress(
|
||||
dbc.Progress(
|
||||
value=pct, label=f"{pct} %", bar=True, color=color, style={"color": "white"}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_considerations_card_content(lff: pl.LazyFrame) -> html.Div:
|
||||
"""Trois barres : champs renseignés + part positive pour sociales et environnementales."""
|
||||
stats = compute_considerations_stats(lff)
|
||||
|
||||
count_ren, pct_ren = stats["champs_renseignes"]
|
||||
blocks = [
|
||||
html.Div(
|
||||
className="mb-3",
|
||||
children=[
|
||||
html.Div(
|
||||
className="d-flex justify-content-between",
|
||||
children=[
|
||||
html.Span("Champs considérations renseignés"),
|
||||
html.Span(
|
||||
f"{format_number(count_ren)} marchés",
|
||||
className="text-muted",
|
||||
),
|
||||
],
|
||||
),
|
||||
_progress_bar(pct_ren, "#6c757d"),
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
for key, label, color in CONSIDERATIONS_RENSEIGNEES:
|
||||
total_count, count, pct = stats[key]
|
||||
blocks.append(
|
||||
html.Div(
|
||||
className="mb-3",
|
||||
children=[
|
||||
html.Div(
|
||||
className="d-flex justify-content-between",
|
||||
children=[
|
||||
html.Span(label),
|
||||
html.Span(
|
||||
f"dans {format_number(count)} marchés",
|
||||
className="text-muted",
|
||||
),
|
||||
],
|
||||
),
|
||||
_progress_bar(pct, color),
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
return html.Div(children=blocks)
|
||||
|
||||
|
||||
def make_card(
|
||||
title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4
|
||||
) -> dbc.Col:
|
||||
@@ -833,13 +988,17 @@ def get_top_org_table(data, org_type: str, extra_columns: list, filters: bool =
|
||||
lff = lff.cast(pl.String)
|
||||
lff = lff.fill_null("")
|
||||
|
||||
dff: pl.DataFrame = lff.collect(engine="streaming")
|
||||
try:
|
||||
dff: pl.DataFrame = lff.collect(engine="streaming")
|
||||
except ColumnNotFoundError:
|
||||
logger.warning(f"get_top_org_table: column not found. {lff.collect_schema()}")
|
||||
return html.Div()
|
||||
|
||||
if dff.height == 0:
|
||||
return html.Div()
|
||||
|
||||
columns, tooltip = setup_table_columns(
|
||||
dff, hideable=False, exclude=[f"{org_type}_id"], new_columns=["Attributions"]
|
||||
dff, hideable=False, exclude=[f"{org_type}_id"]
|
||||
)
|
||||
dff = add_links(dff)
|
||||
data = dff.to_dicts()
|
||||
|
||||
+16
-8
@@ -49,6 +49,16 @@ Vous pouvez consommer les données qui alimentent decp.info
|
||||
|
||||
- en les téléchargeant [sur data.gouv.fr](https://www.data.gouv.fr/datasets/donnees-essentielles-de-la-commande-publique-consolidees-format-tabulaire) (Parquet, CSV), pensez à lire la description du jeu de données
|
||||
- en interrogeant l'[API REST ouverte](https://www.data.gouv.fr/datasets/donnees-essentielles-de-la-commande-publique-consolidees-format-tabulaire#user-content-api-rest)
|
||||
"""
|
||||
),
|
||||
html.H4("API privée", id="api-privee"),
|
||||
dcc.Markdown(
|
||||
"""
|
||||
Une API HTTP est disponible pour accéder aux mêmes données par programme.
|
||||
Documentation interactive : [Swagger UI](/api/v1/swagger).
|
||||
|
||||
L'accès se fait sur token. Pour en obtenir un, contactez
|
||||
[colin@maudry.com](mailto:colin@maudry.com).
|
||||
"""
|
||||
),
|
||||
html.H4("Contact", id="contact"),
|
||||
@@ -87,14 +97,7 @@ Vous pouvez consommer les données qui alimentent decp.info
|
||||
dcc.Markdown(
|
||||
"""Les données visibles sur ce site proviennent exclusivement de la publication de données ouvertes par les acheteurs publics ou en leur nom, régie par [l'arrêté du 22 décembre 2022](https://www.legifrance.gouv.fr/jorf/id/JORFTEXT000046850496). Leur qualité est donc principalement liée à la qualité de leur saisie par les agents publics, parfois peu aidé·es par la qualité des outils à leur disposition. Je pense que l'analyse de marchés individuels et le comptage de marchés sur des critères autres que financiers sont plutôt fiables. En revanche, certains montants de marché estimés à des valeurs farfelues ([1 euro](https://decp.info/marches/432766947000192025S01301), [1 milliard](https://decp.info/marches/2459004280001320210000000271)) faussent les calculs par aggrégation (sommes, moyennes, médianes) et donc la production de statistiques financières fiables. Acheteurs, acheteuses : s'il vous plaît, essayez d'estimer les montants des marchés publics attribués de manière plus précise.
|
||||
|
||||
Quant à l'exhaustivité, je consolide toutes les sources de données exploitables que j'ai pu identifier (voir [ci-dessous](/a-propos#sources). Certains profils d'acheteurs ne publient pas leurs données malgré l'obligation réglementaire :
|
||||
|
||||
- klekoon.fr (ils y travaillent)
|
||||
- safetender.com (Omnikles)
|
||||
|
||||
**marches-publics.info** (AWS) publie ses données de manière assez sporadique depuis début 2023. Compte tenu de son poids dans le secteur, c'est assez dommageable pour la transparence des marchés publics.
|
||||
|
||||
Au milieu de ces mauvaises nouvelles, je tiens à souligner la belle continuité de la publication par la DGFiP des données des marchés publics remontées via le [protocole PES](https://www.collectivites-locales.gouv.fr/finances-locales/le-protocole-dechange-standard-pes). Merci à leurs équipes."""
|
||||
Quant à l'exhaustivité, je consolide toutes les sources de données exploitables que j'ai pu identifier (voir [ci-dessous](/bin.usr-is-merged/)). Je tiens à souligner la belle continuité de la publication par la DGFiP des données des marchés publics remontées via le [protocole PES](https://www.collectivites-locales.gouv.fr/finances-locales/le-protocole-dechange-standard-pes). Merci à leurs équipes."""
|
||||
),
|
||||
html.H4("Sources de données ", id="sources"),
|
||||
get_sources_tables(os.getenv("SOURCE_STATS_CSV_PATH")),
|
||||
@@ -156,6 +159,11 @@ J'enregistre également les données suivantes, de manière anonyme, afin de mie
|
||||
href="#donnees-brutes",
|
||||
className="toc-link",
|
||||
),
|
||||
html.A(
|
||||
"API privée",
|
||||
href="#api-privee",
|
||||
className="toc-link",
|
||||
),
|
||||
html.A(
|
||||
"Contact", href="#contact", className="toc-link"
|
||||
),
|
||||
|
||||
+14
-4
@@ -34,6 +34,7 @@ from src.utils.table import (
|
||||
get_default_hidden_columns,
|
||||
prepare_table_data,
|
||||
sort_table_data,
|
||||
write_styled_excel,
|
||||
)
|
||||
from src.utils.tracking import track_search
|
||||
|
||||
@@ -257,8 +258,15 @@ def update_acheteur_infos(url):
|
||||
if data_etablissement:
|
||||
data_etablissement = data_etablissement[0]
|
||||
|
||||
# Extraction du code département à partir du code postal
|
||||
code_postal = data_etablissement.get("code_postal", "")
|
||||
departement_code = code_postal[:2] if code_postal else None
|
||||
|
||||
# Création de la carte avec le code département pour un centrage approprié
|
||||
acheteur_map = point_on_map(
|
||||
data_etablissement["latitude"], data_etablissement["longitude"]
|
||||
data_etablissement["latitude"],
|
||||
data_etablissement["longitude"],
|
||||
departement_code,
|
||||
)
|
||||
code_departement, nom_departement, nom_region = get_departement_region(
|
||||
data_etablissement["code_postal"]
|
||||
@@ -390,8 +398,10 @@ def download_acheteur_data(
|
||||
df_to_download = pl.DataFrame(data)
|
||||
|
||||
def to_bytes(buffer):
|
||||
df_to_download.write_excel(
|
||||
buffer, worksheet="DECP" if annee in ["Toutes les années", None] else annee
|
||||
write_styled_excel(
|
||||
df_to_download,
|
||||
buffer,
|
||||
worksheet="DECP" if annee in ["Toutes les années", None] else annee,
|
||||
)
|
||||
|
||||
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
@@ -432,7 +442,7 @@ def download_filtered_acheteur_data(
|
||||
lff = sort_table_data(lff, sort_by)
|
||||
|
||||
def to_bytes(buffer):
|
||||
lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
|
||||
write_styled_excel(lff.collect(engine="streaming"), buffer)
|
||||
|
||||
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
return dcc.send_bytes(
|
||||
|
||||
@@ -2,6 +2,7 @@ import polars as pl
|
||||
from dash import Input, Output, callback, dcc, html, register_page
|
||||
|
||||
from src.db import get_cursor
|
||||
from src.utils import logger
|
||||
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
|
||||
|
||||
NAME = "Liste des marchés publics"
|
||||
@@ -27,9 +28,12 @@ def make_org_nom_verbe(org_type, org_id) -> tuple:
|
||||
|
||||
|
||||
def get_title(code, org_type, org_id):
|
||||
org_nom, verbe = make_org_nom_verbe(org_type, org_id)
|
||||
|
||||
return f"Marchés publics {verbe} par {org_nom} | decp.info"
|
||||
if org_type:
|
||||
org_nom, verbe = make_org_nom_verbe(org_type, org_id)
|
||||
return f"Marchés publics {verbe} par {org_nom} | decp.info"
|
||||
else:
|
||||
logger.warning(f"Pas de org_type pour org_id: {org_id}")
|
||||
return "Marchés publics | decp.info"
|
||||
|
||||
|
||||
def get_description(code, org_type, org_id):
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
from dash import Input, Output, State, callback, ctx, dcc, html, register_page
|
||||
|
||||
from src.utils.seo import META_CONTENT
|
||||
|
||||
NAME = "Quelles données pour quelles étapes et quels seuils dans les marchés publics ?"
|
||||
|
||||
register_page(
|
||||
__name__,
|
||||
path="/etapes",
|
||||
title=f"{NAME} | decp.info",
|
||||
name="Étapes et données",
|
||||
description=(
|
||||
"À chaque étape d'un marché public (programmation, publicité, "
|
||||
"attribution), quelles données sont publiées et à partir de quel "
|
||||
"seuil : DECP, BOAMP, JOUE, journaux d'annonces légales, Approch."
|
||||
),
|
||||
image_url=META_CONTENT["image_url"],
|
||||
)
|
||||
|
||||
# Contenu des fiches — à rédiger en Markdown.
|
||||
# Clés barres : "bar-approch", "bar-jal", "bar-boamp", "bar-joue-marche",
|
||||
# "bar-decp", "bar-joue-attribution"
|
||||
# Clés étapes (mobile) : "stage-programmation", "stage-publicite",
|
||||
# "stage-attribution", "stage-contrat", "stage-paiement"
|
||||
ALL_CONTENT: dict[str, str | None] = {
|
||||
"bar-approch": None,
|
||||
"bar-jal": None,
|
||||
"bar-boamp": None,
|
||||
"bar-joue-marche": None,
|
||||
"bar-decp": None,
|
||||
"bar-joue-attribution": None,
|
||||
"stage-programmation": None,
|
||||
"stage-publicite": None,
|
||||
"stage-attribution": None,
|
||||
"stage-contrat": None,
|
||||
"stage-paiement": None,
|
||||
}
|
||||
|
||||
_BAR_IDS = [
|
||||
"bar-approch",
|
||||
"bar-jal",
|
||||
"bar-boamp",
|
||||
"bar-joue-marche",
|
||||
"bar-decp",
|
||||
"bar-joue-attribution",
|
||||
]
|
||||
_STAGE_IDS = [
|
||||
"stage-programmation",
|
||||
"stage-publicite",
|
||||
"stage-attribution",
|
||||
"stage-contrat",
|
||||
"stage-paiement",
|
||||
]
|
||||
|
||||
|
||||
def _lane(*bars):
|
||||
"""Une ligne d'étape : fond segmenté en 5 + barres positionnées."""
|
||||
return html.Div(
|
||||
className="etapes-lane",
|
||||
children=[
|
||||
html.Div(
|
||||
className="etapes-segs",
|
||||
children=[html.Div() for _ in range(5)],
|
||||
),
|
||||
*bars,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _bar(label, color, style, bar_id=None):
|
||||
base = {"backgroundColor": color}
|
||||
base.update(style)
|
||||
props = {"className": "etapes-bar", "style": base}
|
||||
if bar_id is not None:
|
||||
props["id"] = bar_id
|
||||
props["n_clicks"] = 0
|
||||
return html.Div(label, **props)
|
||||
|
||||
|
||||
def build_chart():
|
||||
return html.Div(
|
||||
className="etapes-chart-scroll",
|
||||
children=html.Div(
|
||||
className="etapes-chart",
|
||||
children=[
|
||||
# En-tête : coin vide + 5 marqueurs de seuils
|
||||
html.Div(className="etapes-corner"),
|
||||
html.Div(
|
||||
className="etapes-xhead",
|
||||
children=[
|
||||
html.Div("0 €", className="etapes-xcell"),
|
||||
html.Div(
|
||||
[html.Strong("40 000 €"), "seuil DECP"],
|
||||
className="etapes-xcell",
|
||||
),
|
||||
html.Div(
|
||||
[html.Strong("90 000 €"), "publicité"],
|
||||
className="etapes-xcell",
|
||||
),
|
||||
html.Div(
|
||||
[html.Strong("140 k€ / 216 k€"), "seuils formalisés (UE)"],
|
||||
className="etapes-xcell",
|
||||
),
|
||||
html.Div(
|
||||
[html.Strong("5,404 M€"), "travaux (UE)"],
|
||||
className="etapes-xcell",
|
||||
),
|
||||
],
|
||||
),
|
||||
# Programmation
|
||||
html.Div("Programmation", className="etapes-stage"),
|
||||
_lane(
|
||||
_bar(
|
||||
"Approch — sourcing / préinformation (non réglementaire)",
|
||||
"#7c5cff",
|
||||
{"left": "2%", "right": "2%"},
|
||||
bar_id="bar-approch",
|
||||
),
|
||||
),
|
||||
# Publicité (appel d'offres)
|
||||
html.Div(["Publicité"], className="etapes-stage"),
|
||||
_lane(
|
||||
_bar(
|
||||
"JAL",
|
||||
"#f79009",
|
||||
{"left": "40%", "right": "40%", "top": "6px", "height": "20px"},
|
||||
bar_id="bar-jal",
|
||||
),
|
||||
_bar(
|
||||
"BOAMP",
|
||||
"#1570ef",
|
||||
{"left": "40%", "right": "2%", "top": "28px", "height": "20px"},
|
||||
bar_id="bar-boamp",
|
||||
),
|
||||
_bar(
|
||||
"JOUE — avis de marché",
|
||||
"#0e9384",
|
||||
{"left": "60%", "right": "2%", "top": "6px", "height": "20px"},
|
||||
bar_id="bar-joue-marche",
|
||||
),
|
||||
),
|
||||
# Attribution
|
||||
html.Div("Attribution", className="etapes-stage"),
|
||||
_lane(
|
||||
_bar(
|
||||
"DECP — données essentielles",
|
||||
"#12b76a",
|
||||
{"left": "20%", "right": "2%", "top": "6px", "height": "20px"},
|
||||
bar_id="bar-decp",
|
||||
),
|
||||
_bar(
|
||||
"JOUE — avis d'attribution",
|
||||
"#0e9384",
|
||||
{"left": "60%", "right": "2%", "top": "28px", "height": "20px"},
|
||||
bar_id="bar-joue-attribution",
|
||||
),
|
||||
),
|
||||
# Contrat (vide)
|
||||
html.Div("Contrat", className="etapes-stage"),
|
||||
html.Div(
|
||||
"— aucune donnée publiée aujourd'hui —",
|
||||
className="etapes-lane etapes-empty",
|
||||
),
|
||||
# Paiement (vide)
|
||||
html.Div("Paiement", className="etapes-stage"),
|
||||
html.Div(
|
||||
"— aucune donnée publiée aujourd'hui —",
|
||||
className="etapes-lane etapes-empty",
|
||||
),
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Données par étape, partagées par la vue mobile.
|
||||
# Chaque tuple : (libellé étape, id CSS, [(libellé, couleur, plage seuils)]).
|
||||
STAGES_MOBILE = [
|
||||
(
|
||||
"Programmation",
|
||||
"stage-programmation",
|
||||
[
|
||||
("Approch", "#7c5cff", "tous montants — publication non réglementaire"),
|
||||
],
|
||||
),
|
||||
(
|
||||
"Publicité (appel d'offres)",
|
||||
"stage-publicite",
|
||||
[
|
||||
("JAL", "#f79009", "de 90 000 € au seuil formalisé"),
|
||||
("BOAMP", "#1570ef", "à partir de 90 000 €"),
|
||||
(
|
||||
"JOUE — avis de marché",
|
||||
"#0e9384",
|
||||
"à partir des seuils formalisés (140 k€ / 216 k€)",
|
||||
),
|
||||
],
|
||||
),
|
||||
(
|
||||
"Attribution",
|
||||
"stage-attribution",
|
||||
[
|
||||
("DECP — données essentielles", "#12b76a", "à partir de 40 000 €"),
|
||||
("JOUE — avis d'attribution", "#0e9384", "à partir des seuils formalisés"),
|
||||
],
|
||||
),
|
||||
("Contrat", "stage-contrat", []),
|
||||
("Paiement", "stage-paiement", []),
|
||||
]
|
||||
|
||||
|
||||
def build_mobile():
|
||||
blocks = []
|
||||
for stage, stage_id, items in STAGES_MOBILE:
|
||||
if items:
|
||||
children = [
|
||||
html.Div(
|
||||
[
|
||||
html.I(style={"backgroundColor": color}),
|
||||
html.Span(label, className="etapes-m-label"),
|
||||
html.Span(seuil, className="etapes-m-seuil"),
|
||||
],
|
||||
className="etapes-m-item",
|
||||
)
|
||||
for label, color, seuil in items
|
||||
]
|
||||
else:
|
||||
children = [
|
||||
html.Div(
|
||||
"aucune donnée publiée aujourd'hui",
|
||||
className="etapes-m-item etapes-m-empty",
|
||||
)
|
||||
]
|
||||
blocks.append(
|
||||
html.Div(
|
||||
[
|
||||
html.Div(
|
||||
[
|
||||
html.H4(stage, className="etapes-m-stage"),
|
||||
html.Button(
|
||||
"Voir fiche →",
|
||||
id=stage_id,
|
||||
n_clicks=0,
|
||||
className="etapes-m-link",
|
||||
),
|
||||
],
|
||||
className="etapes-m-header",
|
||||
),
|
||||
*children,
|
||||
],
|
||||
className="etapes-m-block",
|
||||
)
|
||||
)
|
||||
return html.Div(blocks, className="etapes-mobile")
|
||||
|
||||
|
||||
layout = html.Div(
|
||||
className="container",
|
||||
children=[
|
||||
html.H2(NAME),
|
||||
dcc.Markdown(
|
||||
"Un marché public passe par plusieurs étapes. À chacune, des "
|
||||
"données peuvent être publiées — selon le montant du marché et "
|
||||
"des obligations réglementaires. Ce graphique situe les "
|
||||
"principales publications de données par **étape** (de haut en "
|
||||
"bas) et par **seuil** (de gauche à droite, en euros hors taxes)."
|
||||
),
|
||||
build_chart(),
|
||||
build_mobile(),
|
||||
dcc.Store(id="etapes-selected", data=None),
|
||||
html.Div(id="etapes-detail", className="etapes-detail"),
|
||||
dcc.Markdown(
|
||||
"**À noter :** l'axe horizontal n'est pas linéaire — les seuils "
|
||||
"sont espacés régulièrement pour rester lisibles. Les étapes "
|
||||
"*Contrat* et *Paiement* n'ont aujourd'hui aucune donnée publiée "
|
||||
"en open data.",
|
||||
className="etapes-note",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@callback(
|
||||
Output("etapes-detail", "children"),
|
||||
Output("etapes-selected", "data"),
|
||||
[Input(id_, "n_clicks") for id_ in _BAR_IDS + _STAGE_IDS],
|
||||
State("etapes-selected", "data"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def _show_detail(*args):
|
||||
current = args[-1]
|
||||
triggered = ctx.triggered_id
|
||||
if triggered == current:
|
||||
return None, None
|
||||
content = ALL_CONTENT.get(triggered)
|
||||
if content is None:
|
||||
return dcc.Markdown(f"*Fiche en cours de rédaction.* {triggered}"), triggered
|
||||
return dcc.Markdown(content), triggered
|
||||
+3
-2
@@ -109,7 +109,7 @@ def update_marche_info(marche, titulaires):
|
||||
column_object = DATA_SCHEMA.get(col)
|
||||
column_name = column_object.get("title") if column_object else col
|
||||
|
||||
if marche[col]:
|
||||
if marche and col in marche:
|
||||
if col == "acheteur_nom":
|
||||
value = html.A(
|
||||
href=f"/acheteurs/{marche['acheteur_id']}",
|
||||
@@ -134,6 +134,7 @@ def update_marche_info(marche, titulaires):
|
||||
"considerationsSociales",
|
||||
"considerationsEnvironnementales",
|
||||
]
|
||||
and col in marche
|
||||
and "," in marche[col]
|
||||
):
|
||||
col_values = marche[col].split(", ")
|
||||
@@ -243,7 +244,7 @@ def get_marche_jsonld(marche, titulaires) -> str:
|
||||
titulaire.get("titulaire_id"),
|
||||
org_name=titulaire.get("titulaire_nom"),
|
||||
org_type="titulaire",
|
||||
type_org_id=titulaire.get("titulaire_typeIdentifiant"),
|
||||
type_org_id=titulaire.get("titulaire_typeIdentifiant", "SIRET"),
|
||||
),
|
||||
"orderedItem": {
|
||||
"@type": type_order,
|
||||
|
||||
+47
-22
@@ -16,10 +16,11 @@ from dash import (
|
||||
register_page,
|
||||
)
|
||||
|
||||
from src.db import query_marches, schema
|
||||
from src.db import schema
|
||||
from src.figures import (
|
||||
DataTable,
|
||||
get_barchart_sources,
|
||||
get_considerations_card_content,
|
||||
get_dashboard_summary_table,
|
||||
get_distance_histogram,
|
||||
get_duplicate_matrix,
|
||||
@@ -39,7 +40,12 @@ from src.utils.data import (
|
||||
)
|
||||
from src.utils.frontend import get_enum_values_as_dict
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import COLUMNS, get_default_hidden_columns, prepare_table_data
|
||||
from src.utils.table import (
|
||||
COLUMNS,
|
||||
get_default_hidden_columns,
|
||||
prepare_table_data,
|
||||
write_styled_excel,
|
||||
)
|
||||
|
||||
NAME = "Observatoire"
|
||||
|
||||
@@ -508,17 +514,26 @@ Alors, on fait comment ?
|
||||
size="xl",
|
||||
),
|
||||
# DataTable
|
||||
html.Div(
|
||||
className="marches_table",
|
||||
children=DataTable(
|
||||
dtid="observatoire-preview-table",
|
||||
page_size=5,
|
||||
page_action="custom",
|
||||
sort_action="custom",
|
||||
filter_action="custom",
|
||||
hidden_columns=[],
|
||||
columns=[{"id": col, "name": col} for col in OBSERVATOIRE_COLUMNS],
|
||||
),
|
||||
dcc.Loading(
|
||||
overlay_style={"visibility": "visible", "filter": "blur(2px)"},
|
||||
id="loading-statistques",
|
||||
type="default",
|
||||
children=[
|
||||
html.Div(
|
||||
className="marches_table",
|
||||
children=DataTable(
|
||||
dtid="observatoire-preview-table",
|
||||
page_size=5,
|
||||
page_action="custom",
|
||||
sort_action="custom",
|
||||
filter_action="custom",
|
||||
hidden_columns=[],
|
||||
columns=[
|
||||
{"id": col, "name": col} for col in OBSERVATOIRE_COLUMNS
|
||||
],
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
],
|
||||
),
|
||||
@@ -664,10 +679,8 @@ def _compute_dashboard_children(filter_params_normalized: tuple):
|
||||
k: (list(v) if isinstance(v, tuple) else v) for k, v in filter_params_normalized
|
||||
}
|
||||
|
||||
lff: pl.LazyFrame = query_marches().lazy()
|
||||
lff = prepare_dashboard_data(lff=lff, **filter_params)
|
||||
|
||||
dff = lff.collect(engine="streaming")
|
||||
dff = prepare_dashboard_data(**filter_params)
|
||||
lff = dff.lazy()
|
||||
|
||||
df_per_uid = (
|
||||
dff.select("uid", "montant").group_by("uid").agg(pl.col("montant").first())
|
||||
@@ -715,6 +728,15 @@ def _compute_dashboard_children(filter_params_normalized: tuple):
|
||||
)
|
||||
)
|
||||
|
||||
considerations_content = get_considerations_card_content(lff)
|
||||
cards.append(
|
||||
make_card(
|
||||
title="Considérations sociales et environnementales",
|
||||
subtitle="part des marchés concernés",
|
||||
fig=considerations_content,
|
||||
)
|
||||
)
|
||||
|
||||
distance_histogram = get_distance_histogram(lff)
|
||||
cards.append(
|
||||
make_card(
|
||||
@@ -788,13 +810,13 @@ def update_dashboard_cards(*filter_values):
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def download_observatoire(_n_clicks, filter_params, hidden_columns):
|
||||
lff = prepare_dashboard_data(lff=query_marches().lazy(), **(filter_params or {}))
|
||||
dff = prepare_dashboard_data(**(filter_params or {}))
|
||||
|
||||
if hidden_columns:
|
||||
lff = lff.drop(hidden_columns)
|
||||
dff = dff.drop(hidden_columns)
|
||||
|
||||
def to_bytes(buffer):
|
||||
lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
|
||||
write_styled_excel(dff, buffer)
|
||||
|
||||
date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
return dcc.send_bytes(to_bytes, filename=f"decp_observatoire_{date}.xlsx")
|
||||
@@ -817,6 +839,9 @@ def toggle_montant_modal(n_triggers, _close):
|
||||
prevent_initial_call=False,
|
||||
)
|
||||
def add_organization_name_in_title(acheteur_id, titulaire_id):
|
||||
acheteur_id = acheteur_id.replace(" ", "") if acheteur_id else None
|
||||
titulaire_id = titulaire_id.replace(" ", "") if titulaire_id else None
|
||||
|
||||
def lookup_nom(df_org, id_col, nom_col, org_id):
|
||||
match = df_org.filter(pl.col(id_col) == org_id)
|
||||
return match[nom_col].item(0) if match.height >= 1 else None
|
||||
@@ -879,10 +904,10 @@ def populate_preview_table(
|
||||
if not is_open:
|
||||
return (no_update,) * 9
|
||||
|
||||
lff = prepare_dashboard_data(lff=query_marches().lazy(), **(filter_params or {}))
|
||||
dff = prepare_dashboard_data(**(filter_params or {}))
|
||||
|
||||
return prepare_table_data(
|
||||
lff,
|
||||
dff.lazy(),
|
||||
data_timestamp,
|
||||
filter_query,
|
||||
page_current,
|
||||
|
||||
+26
-13
@@ -21,7 +21,7 @@ from dash import (
|
||||
|
||||
from src.db import query_marches, schema
|
||||
from src.figures import DataTable, make_column_picker
|
||||
from src.utils import logger
|
||||
from src.utils import get_data_update_timestamp, logger
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import (
|
||||
COLUMNS,
|
||||
@@ -30,12 +30,20 @@ from src.utils.table import (
|
||||
invert_columns,
|
||||
prepare_table_data,
|
||||
sort_table_data,
|
||||
write_styled_excel,
|
||||
)
|
||||
from src.utils.tracking import track_search
|
||||
|
||||
update_date_timestamp = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH"))
|
||||
update_date = datetime.fromtimestamp(update_date_timestamp).strftime("%d/%m/%Y")
|
||||
update_date_iso = datetime.fromtimestamp(update_date_timestamp).isoformat()
|
||||
update_date_timestamp = get_data_update_timestamp(
|
||||
os.getenv("DATA_FILE_PARQUET_PATH", ""),
|
||||
os.getenv("DUCKDB_PATH", "./decp.duckdb"),
|
||||
)
|
||||
if update_date_timestamp is not None:
|
||||
update_date = datetime.fromtimestamp(update_date_timestamp).strftime("%d/%m/%Y")
|
||||
update_date_iso = datetime.fromtimestamp(update_date_timestamp).isoformat()
|
||||
else:
|
||||
update_date = "date inconnue"
|
||||
update_date_iso = ""
|
||||
|
||||
|
||||
NAME = "Tableau"
|
||||
@@ -117,7 +125,11 @@ layout = [
|
||||
"contentUrl": "https://www.data.gouv.fr/api/1/datasets/r/11cea8e8-df3e-4ed1-932b-781e2635e432",
|
||||
},
|
||||
],
|
||||
"temporalCoverage": f"2018-01-01/{update_date_iso[:10]}",
|
||||
**(
|
||||
{"temporalCoverage": f"2018-01-01/{update_date_iso[:10]}"}
|
||||
if update_date_iso
|
||||
else {}
|
||||
),
|
||||
"spatialCoverage": {
|
||||
"@type": "Place",
|
||||
"address": {"countryCode": "FR"},
|
||||
@@ -163,18 +175,19 @@ layout = [
|
||||
|
||||
Vous pouvez appliquer un filtre pour chaque colonne en entrant du texte sous le nom de la colonne, puis en tapant sur `Entrée`.
|
||||
|
||||
- Champs textuels : la recherche retourne les valeurs qui contiennent le texte recherché et n'est pas sensible à la casse (majuscules/minuscules).
|
||||
- Exemple : `rennes` retourne "RENNES METROPOLE".
|
||||
- Champs textuels : la recherche retourne les valeurs qui contiennent le texte recherché, n'est sensible ni à la casse (majuscules/minuscules), ni à l'accentuation.
|
||||
- `rennes` => le texte contient "rennes"
|
||||
- `metro* *pole` => le texte contient un mot qui commence par "metro" et un mot qui finit par "pole"
|
||||
- `metropole rennes` => le texte contient les mots "metropole" et "rennes", n'importe où dans le texte
|
||||
- `métropole+rennes` => le texte contient "metropole rennes" ou "métropole rennes", collé et dans cet ordre
|
||||
- `metropole+rennes travaux distri*` => le texte contient "metropole rennes", "travaux" et un mot qui commence par "distri"
|
||||
- Les guillemets simples (apostrophe du 4) doivent être prédédées d'une barre oblique (AltGr + 8). Exemple : `services d\\\'assurances`
|
||||
- Champs numériques (Durée en mois, Montant, ...) : vous pouvez...
|
||||
- soit taper un nombre pour trouver les valeurs strictement égales. Exemple : `12` ne retourne que des 12
|
||||
- soit le précéder de **>** ou **<** pour filtrer les valeurs supérieures ou inférieures. Exemple pour les offres reçues : `> 4` retourne les marchés ayant reçu plus de 4 offres.
|
||||
- Champs date (Date de notification, ...) : vous pouvez également utiliser **>** ou **<**. Exemples :
|
||||
- Champs date (Date de notification, ...) :
|
||||
- `< 2024-01-31` pour "avant le 31 janvier 2024"
|
||||
- `2024` pour "en 2024", `> 2022` pour "à partir de 2022".
|
||||
- Pour les champs textuels et les champs dates :
|
||||
- pour chercher du texte qui **commence par** votre texte, entrez `texte*`. C'est par exemple utile pour filtrer des acheteurs ou titulaires par numéro SIREN (`123456789*`) ou les marchés sur une année en particulier (`2024*`)
|
||||
- pour chercher du texte qui **finit par** votre texte, entrez `*texte`
|
||||
- `2024` pour "en 2024", `> 2022` pour "à partir de 2022"
|
||||
|
||||
Vous pouvez filtrer plusieurs colonnes à la fois.
|
||||
|
||||
@@ -330,7 +343,7 @@ def download_data(n_clicks, filter_query, sort_by, hidden_columns: list | None =
|
||||
lff = sort_table_data(lff, sort_by)
|
||||
|
||||
def to_bytes(buffer):
|
||||
lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
|
||||
write_styled_excel(lff.collect(engine="streaming"), buffer)
|
||||
|
||||
date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
return dcc.send_bytes(to_bytes, filename=f"decp_{date}.xlsx")
|
||||
|
||||
+38
-11
@@ -33,6 +33,7 @@ from src.utils.table import (
|
||||
get_default_hidden_columns,
|
||||
prepare_table_data,
|
||||
sort_table_data,
|
||||
write_styled_excel,
|
||||
)
|
||||
from src.utils.tracking import track_search
|
||||
|
||||
@@ -86,13 +87,19 @@ layout = [
|
||||
className="mb-2",
|
||||
children=[
|
||||
dbc.Col(
|
||||
html.H2(
|
||||
children=[
|
||||
html.Span(id="titulaire_siret"),
|
||||
" - ",
|
||||
html.Span(id="titulaire_nom"),
|
||||
],
|
||||
),
|
||||
[
|
||||
html.H2(
|
||||
children=[
|
||||
html.Span(id="titulaire_siret"),
|
||||
" - ",
|
||||
html.Span(id="titulaire_nom"),
|
||||
],
|
||||
),
|
||||
html.P(
|
||||
id="titulaire_activite_libelle",
|
||||
style={"color": "gray", "marginTop": "-10px"},
|
||||
),
|
||||
],
|
||||
width=8,
|
||||
),
|
||||
dbc.Col(
|
||||
@@ -254,17 +261,34 @@ layout = [
|
||||
Output(component_id="titulaire_departement", component_property="children"),
|
||||
Output(component_id="titulaire_region", component_property="children"),
|
||||
Output(component_id="titulaire_lien_annuaire", component_property="href"),
|
||||
Output(component_id="titulaire_activite_libelle", component_property="children"),
|
||||
Input(component_id="titulaire_url", component_property="pathname"),
|
||||
)
|
||||
def update_titulaire_infos(url):
|
||||
titulaire_siret = url.split("/")[-1]
|
||||
if "titulaire_activite_libelle" in DF_TITULAIRES.columns:
|
||||
activite_libelle_row = DF_TITULAIRES.filter(
|
||||
pl.col("titulaire_id") == titulaire_siret
|
||||
).select("titulaire_activite_libelle")
|
||||
activite_libelle = (
|
||||
activite_libelle_row.item(0, 0) if activite_libelle_row.height > 0 else ""
|
||||
)
|
||||
else:
|
||||
activite_libelle = ""
|
||||
data = get_annuaire_data(titulaire_siret)
|
||||
data_etablissement = data.get("matching_etablissements") if data else None
|
||||
if data_etablissement:
|
||||
data_etablissement = data_etablissement[0]
|
||||
|
||||
# Extraction du code département à partir du code postal
|
||||
code_postal = data_etablissement.get("code_postal", "")
|
||||
departement_code = code_postal[:2] if code_postal else None
|
||||
|
||||
# Création de la carte avec le code département pour un centrage approprié
|
||||
titulaire_map = point_on_map(
|
||||
data_etablissement["latitude"], data_etablissement["longitude"]
|
||||
data_etablissement["latitude"],
|
||||
data_etablissement["longitude"],
|
||||
departement_code,
|
||||
)
|
||||
code_departement, nom_departement, nom_region = get_departement_region(
|
||||
data_etablissement["code_postal"]
|
||||
@@ -294,6 +318,7 @@ def update_titulaire_infos(url):
|
||||
departement,
|
||||
nom_region,
|
||||
lien_annuaire,
|
||||
activite_libelle,
|
||||
)
|
||||
|
||||
|
||||
@@ -411,8 +436,10 @@ def download_titulaire_data(
|
||||
df_to_download = pl.DataFrame(data)
|
||||
|
||||
def to_bytes(buffer):
|
||||
df_to_download.write_excel(
|
||||
buffer, worksheet="DECP" if annee in ["Toutes les années", None] else annee
|
||||
write_styled_excel(
|
||||
df_to_download,
|
||||
buffer,
|
||||
worksheet="DECP" if annee in ["Toutes les années", None] else annee,
|
||||
)
|
||||
|
||||
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
@@ -453,7 +480,7 @@ def download_filtered_titulaire_data(
|
||||
lff = sort_table_data(lff, sort_by)
|
||||
|
||||
def to_bytes(buffer):
|
||||
lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
|
||||
write_styled_excel(lff.collect(engine="streaming"), buffer)
|
||||
|
||||
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
return dcc.send_bytes(
|
||||
|
||||
@@ -1,5 +1,29 @@
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from src.utils.cache import cache
|
||||
|
||||
|
||||
@cache.memoize()
|
||||
def get_last_modified(parquet_path: str) -> float:
|
||||
logger.info("Récupération de la date de modification des données...")
|
||||
logging.getLogger("httpx").setLevel("WARNING")
|
||||
if parquet_path.startswith("http"):
|
||||
last_modified = httpx.head(
|
||||
url=parquet_path,
|
||||
follow_redirects=True,
|
||||
).headers["last-modified"]
|
||||
last_modified = datetime.strptime(last_modified, "%a, %d %b %Y %X %Z").strftime(
|
||||
"%s"
|
||||
)
|
||||
return float(last_modified)
|
||||
parquet_local_path = Path(parquet_path)
|
||||
return parquet_local_path.stat().st_mtime
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s %(levelname)-8s %(message)s",
|
||||
@@ -17,3 +41,19 @@ DOMAIN_NAME = (
|
||||
if os.getenv("DEVELOPMENT", "False").lower() == "true"
|
||||
else "decp.info"
|
||||
)
|
||||
|
||||
|
||||
def get_data_update_timestamp(
|
||||
parquet_path: str, fallback_path: str | None = None
|
||||
) -> float | None:
|
||||
"""Date de MAJ des données, best-effort, sans jamais lever (usage au boot)."""
|
||||
try:
|
||||
return get_last_modified(parquet_path)
|
||||
except Exception as e:
|
||||
logger.warning(f"Date de mise à jour des données indisponible ({e})")
|
||||
if fallback_path:
|
||||
try:
|
||||
return os.path.getmtime(fallback_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
+89
-135
@@ -2,18 +2,18 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import httpx
|
||||
import polars as pl
|
||||
from httpx import HTTPError, get
|
||||
|
||||
from src.db import get_cursor, schema
|
||||
from src.db import get_cursor, query_marches, schema
|
||||
from src.utils import logger
|
||||
|
||||
logging.getLogger("httpx").setLevel("WARNING")
|
||||
|
||||
|
||||
def get_annuaire_data(siret: str) -> dict:
|
||||
def get_annuaire_data(siret: str) -> dict | None:
|
||||
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
|
||||
try:
|
||||
response = get(url).raise_for_status()
|
||||
@@ -52,146 +52,92 @@ def get_departements_geojson() -> dict:
|
||||
return geojson
|
||||
|
||||
|
||||
def get_departement_region(code_postal):
|
||||
if code_postal > "97000":
|
||||
code_departement = code_postal[:3]
|
||||
else:
|
||||
code_departement = code_postal[:2]
|
||||
nom_departement = DEPARTEMENTS[code_departement]["departement"]
|
||||
nom_region = DEPARTEMENTS[code_departement]["region"]
|
||||
return code_departement, nom_departement, nom_region
|
||||
def get_departement_region(code_postal: str | None):
|
||||
if code_postal:
|
||||
if code_postal > "97000":
|
||||
code_departement = code_postal[:3]
|
||||
else:
|
||||
code_departement = code_postal[:2]
|
||||
nom_departement = DEPARTEMENTS[code_departement]["departement"]
|
||||
nom_region = DEPARTEMENTS[code_departement]["region"]
|
||||
return code_departement, nom_departement, nom_region
|
||||
return "", "", ""
|
||||
|
||||
|
||||
def _validate_schema(raw) -> dict | None:
|
||||
if (
|
||||
isinstance(raw, dict)
|
||||
and isinstance(raw.get("fields"), list)
|
||||
and raw["fields"]
|
||||
and all(isinstance(c, dict) and "name" in c for c in raw["fields"])
|
||||
):
|
||||
return raw
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_remote_schema(url: str | None) -> dict | None:
|
||||
if not url:
|
||||
return None
|
||||
try:
|
||||
raw = get(url, follow_redirects=True).raise_for_status().json()
|
||||
except (
|
||||
httpx.HTTPError,
|
||||
httpx.TransportError,
|
||||
httpx.TimeoutException,
|
||||
json.JSONDecodeError,
|
||||
) as e:
|
||||
logger.error(f"Schéma distant indisponible ({url}) : {e}")
|
||||
return None
|
||||
return _validate_schema(raw)
|
||||
|
||||
|
||||
def _load_schema_file(path: str) -> dict | None:
|
||||
if not path or not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
with open(path) as f:
|
||||
raw = json.load(f)
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
logger.error(f"Schéma local illisible ({path}) : {e}")
|
||||
return None
|
||||
return _validate_schema(raw)
|
||||
|
||||
|
||||
def _persist_schema_cache(raw: dict, path: str) -> None:
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
tmp = f"{path}.tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(raw, f)
|
||||
os.replace(tmp, path)
|
||||
except (OSError, ValueError) as e:
|
||||
logger.warning(f"Écriture du cache schéma échouée ({path}) : {e}")
|
||||
|
||||
|
||||
def get_data_schema() -> dict:
|
||||
# Récupération du schéma des données tabulaires
|
||||
path = os.getenv("DATA_SCHEMA_PATH")
|
||||
if path.startswith("http"):
|
||||
original_schema: dict = get(
|
||||
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
|
||||
).json()
|
||||
elif os.path.exists(path):
|
||||
with open(path) as f:
|
||||
original_schema: dict = json.load(f)
|
||||
cache_path = os.getenv("DATA_SCHEMA_CACHE", "./schema.cache.json")
|
||||
raw = _fetch_remote_schema(os.getenv("DATA_SCHEMA_PATH"))
|
||||
if raw is not None:
|
||||
_persist_schema_cache(raw, cache_path)
|
||||
else:
|
||||
raise Exception(f"Chemin vers le schéma invalide: {path}")
|
||||
|
||||
new_schema = OrderedDict()
|
||||
|
||||
for col in original_schema["fields"]:
|
||||
new_schema[col["name"]] = col
|
||||
|
||||
return new_schema
|
||||
raw = _load_schema_file(cache_path)
|
||||
if raw is None:
|
||||
raise RuntimeError("Aucun schéma disponible (ni distant ni cache).")
|
||||
return OrderedDict((c["name"], c) for c in raw["fields"])
|
||||
|
||||
|
||||
def prepare_dashboard_data(
|
||||
lff: pl.LazyFrame,
|
||||
dashboard_year=None,
|
||||
dashboard_acheteur_id=None,
|
||||
dashboard_acheteur_categorie=None,
|
||||
dashboard_acheteur_departement_code=None,
|
||||
dashboard_titulaire_id=None,
|
||||
dashboard_titulaire_categorie=None,
|
||||
dashboard_titulaire_departement_code=None,
|
||||
dashboard_marche_type=None,
|
||||
dashboard_marche_objet=None,
|
||||
dashboard_marche_code_cpv=None,
|
||||
dashboard_marche_considerations_sociales=None,
|
||||
dashboard_marche_considerations_environnementales=None,
|
||||
dashboard_marche_techniques=None,
|
||||
dashboard_marche_innovant=None,
|
||||
dashboard_marche_sous_traitance_declaree=None,
|
||||
dashboard_montant_min=None,
|
||||
dashboard_montant_max=None,
|
||||
) -> pl.LazyFrame:
|
||||
if dashboard_year:
|
||||
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
|
||||
else:
|
||||
lff = lff.filter(
|
||||
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
|
||||
)
|
||||
def prepare_dashboard_data(**filter_params) -> pl.DataFrame:
|
||||
"""Exécute la requête DuckDB filtrée pour le tableau de bord.
|
||||
|
||||
if dashboard_acheteur_id:
|
||||
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
|
||||
else:
|
||||
if dashboard_acheteur_categorie:
|
||||
lff = lff.filter(
|
||||
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
|
||||
)
|
||||
if dashboard_acheteur_departement_code:
|
||||
lff = lff.filter(
|
||||
pl.col("acheteur_departement_code").is_in(
|
||||
dashboard_acheteur_departement_code
|
||||
)
|
||||
)
|
||||
Retourne une pl.DataFrame matérialisée uniquement pour le sous-ensemble
|
||||
correspondant aux filtres. Les appelants qui ont besoin d'une LazyFrame
|
||||
appellent `.lazy()` sur le résultat.
|
||||
"""
|
||||
from src.utils.table_sql import dashboard_filters_to_sql
|
||||
|
||||
if dashboard_titulaire_id:
|
||||
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
|
||||
else:
|
||||
if dashboard_titulaire_categorie:
|
||||
lff = lff.filter(
|
||||
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
|
||||
)
|
||||
if dashboard_titulaire_departement_code:
|
||||
lff = lff.filter(
|
||||
pl.col("titulaire_departement_code").is_in(
|
||||
dashboard_titulaire_departement_code
|
||||
)
|
||||
)
|
||||
|
||||
if dashboard_marche_type:
|
||||
lff = lff.filter(pl.col("type") == dashboard_marche_type)
|
||||
|
||||
if dashboard_marche_objet:
|
||||
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
|
||||
|
||||
if dashboard_marche_code_cpv:
|
||||
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
|
||||
|
||||
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
|
||||
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
|
||||
|
||||
if (
|
||||
dashboard_marche_sous_traitance_declaree
|
||||
and dashboard_marche_sous_traitance_declaree != "all"
|
||||
):
|
||||
lff = lff.filter(
|
||||
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
|
||||
)
|
||||
|
||||
if dashboard_marche_techniques:
|
||||
lff = lff.filter(
|
||||
pl.col("techniques")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_techniques)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_marche_considerations_sociales:
|
||||
lff = lff.filter(
|
||||
pl.col("considerationsSociales")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_considerations_sociales)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_marche_considerations_environnementales:
|
||||
lff = lff.filter(
|
||||
pl.col("considerationsEnvironnementales")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_considerations_environnementales)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_montant_min is not None:
|
||||
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
|
||||
|
||||
if dashboard_montant_max is not None:
|
||||
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
|
||||
|
||||
return lff
|
||||
where_sql, params = dashboard_filters_to_sql(**filter_params)
|
||||
return query_marches(where_sql=where_sql, params=params)
|
||||
|
||||
|
||||
def build_org_frame(org_type: str) -> pl.DataFrame:
|
||||
@@ -212,3 +158,11 @@ DF_TITULAIRES = build_org_frame("titulaire")
|
||||
DEPARTEMENTS = get_departements()
|
||||
DEPARTEMENTS_GEOJSON = get_departements_geojson()
|
||||
DATA_SCHEMA = get_data_schema()
|
||||
# Colonne virtuelle (dérivée de uid) : lien loupe vers la fiche du marché.
|
||||
# Absente du schéma DuckDB, créée à l'affichage dans postprocess_page().
|
||||
DATA_SCHEMA["marche"] = {
|
||||
"name": "marche",
|
||||
"type": "string",
|
||||
"title": "Marché",
|
||||
"description": "Lien vers la fiche détaillée du marché.",
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dic
|
||||
address = None
|
||||
if type_org_id.lower() == "siret" and len(org_id) == 14:
|
||||
annuaire_data = get_annuaire_data(org_id)
|
||||
if not annuaire_data:
|
||||
return {}
|
||||
annuaire_address = annuaire_data["matching_etablissements"][0]
|
||||
code_postal = annuaire_address["code_postal"]
|
||||
commune = annuaire_address["libelle_commune"]
|
||||
|
||||
+135
-48
@@ -2,10 +2,12 @@ import os
|
||||
import uuid
|
||||
|
||||
import polars as pl
|
||||
import xlsxwriter
|
||||
from dash import no_update
|
||||
from polars import selectors as cs
|
||||
from unidecode import unidecode
|
||||
|
||||
from src.db import query_marches, schema
|
||||
from src.db import count_marches, count_unique_marches, query_marches, schema
|
||||
from src.utils import logger
|
||||
from src.utils.cache import cache
|
||||
from src.utils.data import DATA_SCHEMA
|
||||
@@ -154,6 +156,8 @@ def normalize_sort_by(sort_by) -> tuple:
|
||||
|
||||
|
||||
def format_number(number) -> str:
|
||||
if not number:
|
||||
return ""
|
||||
number = "{:,}".format(number).replace(",", " ")
|
||||
return number
|
||||
|
||||
@@ -213,6 +217,25 @@ def format_values(dff: pl.DataFrame) -> pl.DataFrame:
|
||||
return dff
|
||||
|
||||
|
||||
_ACCENT_REPLACEMENTS = [
|
||||
("[éèêëÉÈÊË]", "e"),
|
||||
("[àâäÀÂÄ]", "a"),
|
||||
("[ùûüÙÛÜ]", "u"),
|
||||
("[îïÎÏ]", "i"),
|
||||
("[ôöÔÖ]", "o"),
|
||||
("[çÇ]", "c"),
|
||||
("[ñÑ]", "n"),
|
||||
("[æÆ]", "ae"),
|
||||
("[œŒ]", "oe"),
|
||||
]
|
||||
|
||||
|
||||
def _deaccent_col(expr: pl.Expr) -> pl.Expr:
|
||||
for pattern, replacement in _ACCENT_REPLACEMENTS:
|
||||
expr = expr.str.replace_all(pattern, replacement)
|
||||
return expr
|
||||
|
||||
|
||||
def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame:
|
||||
_schema = lff.collect_schema()
|
||||
filtering_expressions = filter_query.split(" && ")
|
||||
@@ -254,17 +277,17 @@ def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame:
|
||||
elif operator == "contains":
|
||||
if col_type in ["String", "Date"] and isinstance(filter_value, str):
|
||||
filter_value = filter_value.strip('"')
|
||||
normalized_value = unidecode(filter_value)
|
||||
col_expr = _deaccent_col(pl.col(col_name))
|
||||
if filter_value.endswith("*"):
|
||||
lff = lff.filter(
|
||||
pl.col(col_name).str.starts_with(filter_value[:-1])
|
||||
col_expr.str.starts_with(normalized_value[:-1])
|
||||
)
|
||||
elif filter_value.startswith("*"):
|
||||
lff = lff.filter(
|
||||
pl.col(col_name).str.ends_with(filter_value[1:])
|
||||
)
|
||||
lff = lff.filter(col_expr.str.ends_with(normalized_value[1:]))
|
||||
else:
|
||||
lff = lff.filter(
|
||||
pl.col(col_name).str.contains("(?i)" + filter_value)
|
||||
col_expr.str.contains("(?i)" + normalized_value)
|
||||
)
|
||||
elif col_type.startswith("Int") or col_type.startswith("Float"):
|
||||
lff = lff.filter(pl.col(col_name) == filter_value)
|
||||
@@ -337,11 +360,8 @@ def setup_table_columns(
|
||||
def get_default_hidden_columns(page):
|
||||
if page == "acheteur":
|
||||
displayed_columns = [
|
||||
"uid",
|
||||
"objet",
|
||||
"dateNotification",
|
||||
"titulaire_id",
|
||||
"titulaire_typeIdentifiant",
|
||||
"titulaire_nom",
|
||||
"titulaire_distance",
|
||||
"montant",
|
||||
@@ -350,10 +370,8 @@ def get_default_hidden_columns(page):
|
||||
]
|
||||
elif page == "titulaire":
|
||||
displayed_columns = [
|
||||
"uid",
|
||||
"objet",
|
||||
"dateNotification",
|
||||
"acheteur_id",
|
||||
"acheteur_nom",
|
||||
"titulaire_distance",
|
||||
"montant",
|
||||
@@ -361,9 +379,13 @@ def get_default_hidden_columns(page):
|
||||
"dureeRestanteMois",
|
||||
]
|
||||
elif page == "tableau":
|
||||
displayed_columns = os.getenv("DISPLAYED_COLUMNS")
|
||||
displayed_columns = [
|
||||
c.strip() for c in os.getenv("DISPLAYED_COLUMNS", "").split(",")
|
||||
]
|
||||
else:
|
||||
displayed_columns = os.getenv("DISPLAYED_COLUMNS")
|
||||
displayed_columns = [
|
||||
c.strip() for c in os.getenv("DISPLAYED_COLUMNS", "").split(",")
|
||||
]
|
||||
logger.warning(f"Invalid page: {page}")
|
||||
|
||||
hidden_columns = []
|
||||
@@ -376,32 +398,19 @@ def get_default_hidden_columns(page):
|
||||
return hidden_columns
|
||||
|
||||
|
||||
@cache.memoize()
|
||||
def _load_filter_sort_postprocess(filter_query, sort_by_key):
|
||||
logger.debug(
|
||||
f"Cache miss — recomputing for filter={filter_query!r} sort={sort_by_key!r}"
|
||||
)
|
||||
def postprocess_page(dff: pl.DataFrame) -> pl.DataFrame:
|
||||
"""Post-traitement à appliquer sur une page déjà paginée.
|
||||
|
||||
lff: pl.LazyFrame = query_marches().lazy()
|
||||
|
||||
if filter_query:
|
||||
lff = filter_table_data(lff, filter_query)
|
||||
|
||||
if sort_by_key:
|
||||
sort_by = [
|
||||
{"column_id": col, "direction": direction} for col, direction in sort_by_key
|
||||
]
|
||||
lff = sort_table_data(lff, sort_by)
|
||||
|
||||
dff = table_postprocess(lff)
|
||||
|
||||
return dff
|
||||
|
||||
|
||||
def table_postprocess(lff) -> pl.DataFrame:
|
||||
lff = lff.cast(pl.String)
|
||||
lff = lff.fill_null("")
|
||||
dff: pl.DataFrame = lff.collect()
|
||||
À appeler après la pagination.
|
||||
"""
|
||||
dff = dff.with_columns(pl.all().cast(pl.String).fill_null(""))
|
||||
if "uid" in dff.columns:
|
||||
dff = dff.with_columns(
|
||||
(
|
||||
'<a href="/marches/' + pl.col("uid") + '" title="Voir le marché">🔍</a>'
|
||||
).alias("marche")
|
||||
)
|
||||
dff = dff.select(["marche"] + [c for c in dff.columns if c != "marche"])
|
||||
dff = add_links(dff)
|
||||
if "sourceFile" in dff.columns:
|
||||
dff = add_resource_link(dff)
|
||||
@@ -410,6 +419,48 @@ def table_postprocess(lff) -> pl.DataFrame:
|
||||
return dff
|
||||
|
||||
|
||||
@cache.memoize()
|
||||
def _fetch_page_sql(
|
||||
filter_query: str | None,
|
||||
sort_by_key: tuple,
|
||||
page_current: int,
|
||||
page_size: int,
|
||||
) -> tuple[pl.DataFrame, int, int]:
|
||||
"""Chemin rapide : filtre/tri/pagine dans DuckDB, post-traite la page seule.
|
||||
|
||||
Retourne (page_dataframe_post_traitée, total_count, total_unique_count).
|
||||
"""
|
||||
# Import local pour éviter une dépendance circulaire
|
||||
# (src.utils.table_sql importe split_filter_part depuis src.utils.table).
|
||||
from src.utils.table_sql import filter_query_to_sql, sort_by_to_sql
|
||||
|
||||
logger.debug(
|
||||
f"Cache miss SQL — filter={filter_query!r} sort={sort_by_key!r} "
|
||||
f"page={page_current} size={page_size}"
|
||||
)
|
||||
|
||||
where_sql, params = filter_query_to_sql(filter_query or "", schema)
|
||||
|
||||
sort_by_dash = [
|
||||
{"column_id": col, "direction": direction} for col, direction in sort_by_key
|
||||
]
|
||||
order_by = sort_by_to_sql(sort_by_dash, schema) or None
|
||||
|
||||
total = count_marches(where_sql, params)
|
||||
total_unique = count_unique_marches(where_sql, params)
|
||||
|
||||
page = query_marches(
|
||||
where_sql=where_sql,
|
||||
params=params,
|
||||
order_by=order_by,
|
||||
limit=page_size,
|
||||
offset=page_current * page_size,
|
||||
)
|
||||
|
||||
page = postprocess_page(page)
|
||||
return page, total, total_unique
|
||||
|
||||
|
||||
def prepare_table_data(
|
||||
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
|
||||
):
|
||||
@@ -433,9 +484,13 @@ def prepare_table_data(
|
||||
trigger_cleanup = no_update if source_table == "tableau" else str(uuid.uuid4())
|
||||
|
||||
if data is None:
|
||||
# Probablement car il s'agit de la page Tableau
|
||||
sort_by_key = normalize_sort_by(sort_by)
|
||||
dff: pl.DataFrame = _load_filter_sort_postprocess(
|
||||
filter_query=filter_query, sort_by_key=sort_by_key
|
||||
dff, height, total_unique = _fetch_page_sql(
|
||||
filter_query=filter_query,
|
||||
sort_by_key=sort_by_key,
|
||||
page_current=page_current,
|
||||
page_size=page_size,
|
||||
)
|
||||
else:
|
||||
if isinstance(data, list):
|
||||
@@ -450,24 +505,25 @@ def prepare_table_data(
|
||||
if filter_query:
|
||||
lff = filter_table_data(lff, filter_query)
|
||||
|
||||
df_height = lff.select("uid").collect(engine="streaming")
|
||||
height = df_height.height
|
||||
total_unique = df_height["uid"].n_unique()
|
||||
|
||||
if sort_by and len(sort_by) > 0:
|
||||
lff = sort_table_data(lff, sort_by)
|
||||
|
||||
dff: pl.DataFrame = table_postprocess(lff)
|
||||
|
||||
height = dff.height
|
||||
start_row = page_current * page_size
|
||||
lff = lff.slice(start_row, page_size)
|
||||
dff = lff.collect(engine="streaming")
|
||||
dff: pl.DataFrame = postprocess_page(dff)
|
||||
|
||||
if height > 0:
|
||||
nb_rows = (
|
||||
f"{format_number(height)} lignes "
|
||||
f"({format_number(dff.select('uid').unique().height)} marchés)"
|
||||
f"{format_number(height)} lignes ({format_number(total_unique)} marchés)"
|
||||
)
|
||||
else:
|
||||
nb_rows = "0 lignes (0 marchés)"
|
||||
|
||||
start_row = page_current * page_size
|
||||
dff = dff.slice(start_row, page_size)
|
||||
|
||||
table_columns, tooltip = setup_table_columns(dff)
|
||||
|
||||
dicts = dff.to_dicts()
|
||||
@@ -502,3 +558,34 @@ def invert_columns(columns):
|
||||
|
||||
|
||||
COLUMNS = schema.names()
|
||||
|
||||
_EXCEL_MIN_COLUMN_WIDTH = 132 # ≈ 3.5 cm à 96 DPI
|
||||
_EXCEL_HEADER_FORMAT = {
|
||||
"bold": True,
|
||||
"bg_color": "#b33821",
|
||||
"font_color": "white",
|
||||
}
|
||||
_EXCEL_COLUMN_WIDTHS = {
|
||||
"objet": 350,
|
||||
"acheteur_nom": 250,
|
||||
"titulaire_nom": 250,
|
||||
"acheteur_id": 160,
|
||||
}
|
||||
|
||||
|
||||
def write_styled_excel(df: pl.DataFrame, buffer, worksheet: str = "DECP") -> None:
|
||||
col_widths = {
|
||||
col: max(_EXCEL_MIN_COLUMN_WIDTH, _EXCEL_COLUMN_WIDTHS.get(col, 0))
|
||||
for col in df.columns
|
||||
}
|
||||
wb = xlsxwriter.Workbook(buffer, {"default_format_properties": {"text_wrap": True}})
|
||||
try:
|
||||
ws = wb.add_worksheet(worksheet)
|
||||
df.write_excel(
|
||||
workbook=wb,
|
||||
worksheet=ws,
|
||||
header_format=_EXCEL_HEADER_FORMAT,
|
||||
column_widths=col_widths,
|
||||
)
|
||||
finally:
|
||||
wb.close()
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from src.utils import logger
|
||||
from src.utils.table import split_filter_part
|
||||
|
||||
|
||||
def filter_query_to_sql(filter_query: str, schema: pl.Schema) -> tuple[str, list]:
|
||||
"""Traduit le DSL de filtres de dash_table.DataTable en fragment SQL DuckDB.
|
||||
|
||||
Retourne (where_clause, params) où where_clause est un fragment à injecter
|
||||
après WHERE et params est la liste des valeurs à passer à
|
||||
cursor.execute(sql, params). Les identifiants de colonnes sont validés
|
||||
contre le schéma fourni ; jamais concaténés avec des valeurs utilisateur.
|
||||
"""
|
||||
if not filter_query:
|
||||
return "TRUE", []
|
||||
|
||||
clauses: list[str] = []
|
||||
params: list = []
|
||||
|
||||
for part in filter_query.split(" && "):
|
||||
col_name, operator, raw_value = split_filter_part(part)
|
||||
if not isinstance(col_name, str) or not isinstance(raw_value, str):
|
||||
continue
|
||||
|
||||
if col_name not in schema.names():
|
||||
logger.warning(f"Colonne inconnue ignorée : {col_name!r}")
|
||||
continue
|
||||
|
||||
col_type = schema[col_name]
|
||||
is_numeric = col_type.is_numeric()
|
||||
col_is_date = col_type == pl.Date
|
||||
quoted_col = f'"{col_name}"'
|
||||
|
||||
if is_numeric:
|
||||
try:
|
||||
value = int(raw_value) if col_type.is_integer() else float(raw_value)
|
||||
except ValueError:
|
||||
logger.warning(f"Valeur numérique invalide ignorée : {raw_value!r}")
|
||||
continue
|
||||
|
||||
if operator == "contains":
|
||||
clauses.append(f"{quoted_col} IS NOT NULL AND {quoted_col} = ?")
|
||||
elif operator == ">":
|
||||
clauses.append(f"{quoted_col} IS NOT NULL AND {quoted_col} > ?")
|
||||
elif operator == "<":
|
||||
clauses.append(f"{quoted_col} IS NOT NULL AND {quoted_col} < ?")
|
||||
else:
|
||||
logger.warning(f"Opérateur invalide pour numérique : {operator!r}")
|
||||
continue
|
||||
params.append(value)
|
||||
continue
|
||||
|
||||
# String / Date : toujours traité comme texte (parité avec Polars)
|
||||
value = raw_value.strip('"')
|
||||
|
||||
if operator == "contains":
|
||||
if col_is_date:
|
||||
target = f"CAST({quoted_col} AS VARCHAR)"
|
||||
|
||||
if col_name in ("acheteur_id", "titulaire_id"):
|
||||
value = value.replace(" ", "")
|
||||
where_clause, param_list = tokenize_text_filter(
|
||||
col_name, value, col_is_date
|
||||
)
|
||||
clauses.append(where_clause)
|
||||
params.extend(param_list)
|
||||
logger.debug(params)
|
||||
continue
|
||||
|
||||
elif operator in (">", "<"):
|
||||
target = f"CAST({quoted_col} AS VARCHAR)" if col_is_date else quoted_col
|
||||
clauses.append(f"{quoted_col} IS NOT NULL AND {target} {operator} ?")
|
||||
params.append(value)
|
||||
else:
|
||||
logger.warning(f"Opérateur invalide pour chaîne : {operator!r}")
|
||||
continue
|
||||
|
||||
if not clauses:
|
||||
return "TRUE", []
|
||||
return " AND ".join(clauses), params
|
||||
|
||||
|
||||
def sort_by_to_sql(sort_by: list[dict] | None, schema: pl.Schema) -> str:
|
||||
"""Traduit sort_by (format Dash) en clause ORDER BY DuckDB.
|
||||
|
||||
Retourne '' si pas de tri (aucun ORDER BY à ajouter).
|
||||
"""
|
||||
if not sort_by:
|
||||
return ""
|
||||
|
||||
fragments: list[str] = []
|
||||
for entry in sort_by:
|
||||
col = entry.get("column_id")
|
||||
direction = entry.get("direction")
|
||||
if col not in schema.names():
|
||||
logger.warning(f"Tri sur colonne inconnue ignoré : {col!r}")
|
||||
continue
|
||||
if direction not in ("asc", "desc"):
|
||||
logger.warning(f"Tri sur direction inconnue ignoré : {direction!r}")
|
||||
continue
|
||||
fragments.append(f'"{col}" {direction.upper()} NULLS LAST')
|
||||
|
||||
return ", ".join(fragments)
|
||||
|
||||
|
||||
def dashboard_filters_to_sql(
|
||||
dashboard_year=None,
|
||||
dashboard_acheteur_id=None,
|
||||
dashboard_acheteur_categorie=None,
|
||||
dashboard_acheteur_departement_code=None,
|
||||
dashboard_titulaire_id=None,
|
||||
dashboard_titulaire_categorie=None,
|
||||
dashboard_titulaire_departement_code=None,
|
||||
dashboard_marche_type=None,
|
||||
dashboard_marche_objet=None,
|
||||
dashboard_marche_code_cpv=None,
|
||||
dashboard_marche_considerations_sociales=None,
|
||||
dashboard_marche_considerations_environnementales=None,
|
||||
dashboard_marche_techniques=None,
|
||||
dashboard_marche_innovant=None,
|
||||
dashboard_marche_sous_traitance_declaree=None,
|
||||
dashboard_montant_min=None,
|
||||
dashboard_montant_max=None,
|
||||
) -> tuple[str, list]:
|
||||
"""Traduit les filtres du tableau de bord en (where_clause, params) DuckDB."""
|
||||
clauses: list[str] = []
|
||||
params: list = []
|
||||
|
||||
if dashboard_year:
|
||||
clauses.append('YEAR("dateNotification") = ?')
|
||||
params.append(int(dashboard_year))
|
||||
else:
|
||||
clauses.append('"dateNotification" > ?')
|
||||
params.append(datetime.now() - timedelta(days=365))
|
||||
|
||||
if dashboard_acheteur_id:
|
||||
dashboard_acheteur_id = dashboard_acheteur_id.replace(" ", "")
|
||||
clauses.append('"acheteur_id" LIKE ?')
|
||||
params.append(f"%{dashboard_acheteur_id}%")
|
||||
else:
|
||||
if dashboard_acheteur_categorie:
|
||||
clauses.append('"acheteur_categorie" = ?')
|
||||
params.append(dashboard_acheteur_categorie)
|
||||
if dashboard_acheteur_departement_code:
|
||||
placeholders = ", ".join(["?"] * len(dashboard_acheteur_departement_code))
|
||||
clauses.append(f'"acheteur_departement_code" IN ({placeholders})')
|
||||
params.extend(dashboard_acheteur_departement_code)
|
||||
|
||||
if dashboard_titulaire_id:
|
||||
dashboard_titulaire_id = dashboard_titulaire_id.replace(" ", "")
|
||||
clauses.append('"titulaire_id" LIKE ?')
|
||||
params.append(f"%{dashboard_titulaire_id}%")
|
||||
else:
|
||||
if dashboard_titulaire_categorie:
|
||||
clauses.append('"titulaire_categorie" = ?')
|
||||
params.append(dashboard_titulaire_categorie)
|
||||
if dashboard_titulaire_departement_code:
|
||||
placeholders = ", ".join(["?"] * len(dashboard_titulaire_departement_code))
|
||||
clauses.append(f'"titulaire_departement_code" IN ({placeholders})')
|
||||
params.extend(dashboard_titulaire_departement_code)
|
||||
|
||||
if dashboard_marche_type:
|
||||
clauses.append('"type" = ?')
|
||||
params.append(dashboard_marche_type)
|
||||
|
||||
if dashboard_marche_objet:
|
||||
where_clause, param_list = tokenize_text_filter("objet", dashboard_marche_objet)
|
||||
clauses.append(where_clause)
|
||||
params.extend(param_list)
|
||||
|
||||
if dashboard_marche_code_cpv:
|
||||
clauses.append('"codeCPV" LIKE ?')
|
||||
params.append(f"{dashboard_marche_code_cpv}%")
|
||||
|
||||
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
|
||||
clauses.append('"marcheInnovant" = ?')
|
||||
params.append(dashboard_marche_innovant)
|
||||
|
||||
if (
|
||||
dashboard_marche_sous_traitance_declaree
|
||||
and dashboard_marche_sous_traitance_declaree != "all"
|
||||
):
|
||||
clauses.append('"sousTraitanceDeclaree" = ?')
|
||||
params.append(dashboard_marche_sous_traitance_declaree)
|
||||
|
||||
if dashboard_marche_techniques:
|
||||
clauses.append("list_has_any(string_split(\"techniques\", ', '), ?::VARCHAR[])")
|
||||
params.append(list(dashboard_marche_techniques))
|
||||
|
||||
if dashboard_marche_considerations_sociales:
|
||||
clauses.append(
|
||||
"list_has_any(string_split(\"considerationsSociales\", ', '), ?::VARCHAR[])"
|
||||
)
|
||||
params.append(list(dashboard_marche_considerations_sociales))
|
||||
|
||||
if dashboard_marche_considerations_environnementales:
|
||||
clauses.append(
|
||||
"list_has_any(string_split(\"considerationsEnvironnementales\", ', '), ?::VARCHAR[])"
|
||||
)
|
||||
params.append(list(dashboard_marche_considerations_environnementales))
|
||||
|
||||
if dashboard_montant_min is not None:
|
||||
clauses.append('"montant" >= ?')
|
||||
params.append(dashboard_montant_min)
|
||||
|
||||
if dashboard_montant_max is not None:
|
||||
clauses.append('"montant" <= ?')
|
||||
params.append(dashboard_montant_max)
|
||||
|
||||
return " AND ".join(clauses), params
|
||||
|
||||
|
||||
def tokenize_text_filter(
|
||||
column: str, text: str, col_is_date: bool = False
|
||||
) -> tuple[str, list]:
|
||||
terms = text.split()
|
||||
# si col_is_date alors le deuxième doit être casté en VARCHAR
|
||||
if col_is_date:
|
||||
quoted_col = f'CAST("{column}" AS VARCHAR)'
|
||||
else:
|
||||
quoted_col = f'"{column}"'
|
||||
|
||||
conditions = [f'"{column}" IS NOT NULL', f"{quoted_col} <> ''"]
|
||||
|
||||
params = []
|
||||
|
||||
for term in terms:
|
||||
conditions.append(f"{quoted_col} ILIKE ?")
|
||||
|
||||
if term.startswith("*") or term.endswith("*"):
|
||||
params.append(term.replace("*", "%"))
|
||||
elif "+" in term:
|
||||
params.append(f"%{term.replace('+', ' ')}%")
|
||||
else:
|
||||
params.append(f"%{term}%")
|
||||
|
||||
where_clause = " AND ".join(conditions)
|
||||
return where_clause, params
|
||||
Reference in New Issue
Block a user