colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

Merge branch 'feature/28_vue_acheteur' into dev

Browse Source
This commit is contained in:
Colin Maudry
2025-09-28 00:33:33 +02:00
parent 29428a8706 4b8565dcf2
commit 5122bb7f3a
7 changed files with 530 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -1
View File
@@ -36,7 +36,12 @@ python run.py
## Notes de version
#### 2.0.1 (23 septembre 2025)
#### 2.1.0
- Ajout de la vue acheteur
- Variables globales uniquement en lecture
##### 2.0.1 (23 septembre 2025)
- Bloquage du bouton de téléchargement si trop de lignes (+ 65000) [#38](https://github.com/ColinMaudry/decp.info/issues/38)
- Amélioration du script de déploiement (deploy.sh)
src/app.py
+1
View File
@@ -74,6 +74,7 @@ app.layout = html.Div(
page["name"], href=page["relative_path"], className="nav"
)
for page in page_registry.values()
if page["name"] not in ["Acheteur", "Fournisseur"]
]
),
],
src/assets/style.css
+47 -1
View File
@@ -71,8 +71,12 @@ td[data-dash-column="objet"] {
}
/* Alternance des couleurs pour les lignes */
#table tr:nth-child(even) td {
.marches_table {
font-family: sans-serif;
}
.marches_table .cell-table tr:nth-child(even) td {
background-color: #feeeee;
font-family: sans-serif;
}
#header > *,
@@ -102,9 +106,51 @@ a.nav {
}
h3 {
margin: 36px 0 24px 0;
}
summary > h3 {
margin: 0;
display: inline;
}
#_pages_content {
padding-top: 28px;
}
/* Vue acheteur/fournisseur */
.wrapper {
display: grid;
grid-gap: 10px;
margin-bottom: 50px;
justify-content: space-between;
}
.org_title {
grid-column: 1 / 3;
grid-row: 1;
}
.org_year {
grid-column: 3;
grid-row: 1;
}
.org_infos {
grid-column: 1;
grid-row: 2;
}
.org_infos > p {
margin: 8px 0;
}
.org_stats {
grid-column: 2;
grid-row: 2;
}
.org_map {
grid-column: 3;
grid-row: 2;
}
src/figures.py
+34 -1
View File
@@ -2,7 +2,7 @@ import json
import plotly.express as px
import polars as pl
from dash import dash_table, html
from dash import dash_table, dcc, html
def get_map_count_marches(lf: pl.LazyFrame):
@@ -157,3 +157,36 @@ def get_sources_tables(source_path) -> html.Div:
datatable.data = df.to_dicts()
return html.Div(children=datatable)
def point_on_map(lat, lon):
lat = float(lat)
lon = float(lon)
# Create a scatter mapbox or choropleth map
fig = px.scatter_map(
lat=[lat], lon=[lon], height=300, width=400, color=[1], size=[1]
)
fig.update_coloraxes(showscale=False)
# Set map style (you can use 'open-street-map', 'carto-positron', etc.)
fig.update_layout(
mapbox_style="light", # Light, clean background
margin={"r": 0, "t": 0, "l": 0, "b": 0},
)
# Optionally, center the map on France
fig.update_geos(
center=dict(lat=46.603354, lon=1.888334), # Center of France
lataxis_range=[41, 51.5], # Latitude range for France
lonaxis_range=[-5, 10], # Longitude range for France
)
# But scatter_mapbox doesn't use geos, so better to control via zoom/center manually
# Let's reset and use proper centering in scatter_mapbox instead:
fig.update_layout(map_center={"lat": 46.6, "lon": 1.89}, map_zoom=4)
graph = dcc.Graph(id="map", figure=fig)
return graph
src/pages/acheteur.py
+268
View File
@@ -0,0 +1,268 @@
import datetime
import polars as pl
from dash import Input, Output, State, callback, dash_table, dcc, html, register_page
from src.figures import point_on_map
from src.utils import format_number, get_annuaire_data, get_departement_region, lf
register_page(
__name__,
path_template="/acheteur/<acheteur_id>",
title="decp.info - acheteur",
name="Acheteur",
order=5,
)
# 21690123100011
layout = [
dcc.Store(id="acheteur_data", storage_type="memory"),
dcc.Location(id="url", refresh="callback-nav"),
html.Div(
className="container",
children=[
html.Div(
className="wrapper",
children=[
html.H2(
className="org_title",
children=[
html.Span(id="acheteur_siret"),
" - ",
html.Span(id="acheteur_nom"),
],
),
html.Div(
className="org_year",
children=dcc.Dropdown(
id="acheteur_year",
options=["Toutes"]
+ [
str(year)
for year in range(
2018, int(datetime.date.today().year) + 1
)
],
placeholder="Année",
),
),
html.Div(
className="org_infos",
children=[
# TODO: ajouter le type d'acheteur : commune, CD, CR, etc.
html.P(["Commune : ", html.Strong(id="acheteur_commune")]),
html.P(
[
"Département : ",
html.Strong(id="acheteur_departement"),
]
),
html.P(["Région : ", html.Strong(id="acheteur_region")]),
html.A(
id="acheteur_lien_annuaire",
children="Plus de détails sur l'Annuaire des entreprises",
target="_blank",
),
],
),
html.Div(
className="org_stats",
children=[
html.P(id="acheteur_titre_stats"),
html.P(id="acheteur_marches_attribues"),
html.P(id="acheteur_fournisseurs_differents"),
html.Button(
"Téléchargement au format Excel",
id="btn-download-acheteur-data",
),
dcc.Download(id="download-acheteur-data"),
],
),
html.Div(className="org_map", id="acheteur_map"),
],
),
# récupérer les données de l'acheteur sur l'api annuaire
html.H3("Derniers marchés publics attribués"),
html.Div(id="acheteur_last_marches", children=""),
],
),
]
@callback(
Output(component_id="acheteur_siret", component_property="children"),
Output(component_id="acheteur_nom", component_property="children"),
Output(component_id="acheteur_commune", component_property="children"),
Output(component_id="acheteur_map", component_property="children"),
Output(component_id="acheteur_departement", component_property="children"),
Output(component_id="acheteur_region", component_property="children"),
Output(component_id="acheteur_lien_annuaire", component_property="href"),
Input(component_id="url", component_property="pathname"),
)
def update_acheteur_infos(url):
acheteur_siret = url.split("/")[-1]
if len(acheteur_siret) != 14:
acheteur_siret = (
f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})"
)
data = get_annuaire_data(acheteur_siret)
data_etablissement = data["matching_etablissements"][0]
acheteur_map = point_on_map(
data_etablissement["latitude"], data_etablissement["longitude"]
)
code_departement, nom_departement, nom_region = get_departement_region(
data_etablissement["code_postal"]
)
departement = f"{nom_departement} ({code_departement})"
lien_annuaire = (
f"https://annuaire-entreprises.data.gouv.fr/etablissement/{acheteur_siret}"
)
return (
acheteur_siret,
data["nom_raison_sociale"],
data_etablissement["libelle_commune"],
acheteur_map,
departement,
nom_region,
lien_annuaire,
)
@callback(
Output(component_id="acheteur_marches_attribues", component_property="children"),
Output(
component_id="acheteur_fournisseurs_differents", component_property="children"
),
Input(component_id="acheteur_data", component_property="data"),
)
def update_acheteur_stats(data):
df = pl.DataFrame(data)
if df.height == 0:
df = pl.DataFrame(schema=lf.collect_schema())
df_marches = df.unique("id")
nb_marches = format_number(df_marches.height)
# somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item()))
marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"]
# + ", pour un total de ", html.Strong(somme_marches + " €")]
del df_marches
nb_fournisseurs = df.unique("titulaire_id").height
nb_fournisseurs = [
html.Strong(format_number(nb_fournisseurs)),
" fournisseurs (SIRET) différents",
]
del df
return marches_attribues, nb_fournisseurs
@callback(
Output(component_id="acheteur_data", component_property="data"),
Input(component_id="url", component_property="pathname"),
Input(component_id="acheteur_year", component_property="value"),
)
def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame:
acheteur_siret = url.split("/")[-1]
lff = lf.filter(pl.col("acheteur_id") == acheteur_siret)
lff = lff.fill_null("")
lff = lff.select(
"id",
"objet",
"dateNotification",
"titulaire_id",
"titulaire_nom",
"montant",
"codeCPV",
"dureeMois",
)
if acheteur_year and acheteur_year != "Toutes":
lff = lff.filter(
pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year)
)
lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True)
data = lff.collect(engine="streaming").to_dicts()
return data
@callback(
Output(component_id="acheteur_last_marches", component_property="children"),
Input(component_id="acheteur_data", component_property="data"),
)
def get_last_marches_table(data) -> html.Div:
columns = [
"id",
"objet",
"dateNotification",
"titulaire_nom",
"montant",
"codeCPV",
"dureeMois",
]
table = html.Div(
className="marches_table",
id="marches_datatable",
children=dash_table.DataTable(
data=data,
page_action="native",
columns=[
{
"name": i,
"id": i,
"presentation": "markdown",
"type": "text",
"format": {"nully": "N/A"},
"hideable": False,
}
for i in columns
if i not in ["titulaire_id"]
],
page_size=10,
style_cell_conditional=[
{
"if": {"column_id": "objet"},
"minWidth": "300px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
{
"if": {"column_id": "titulaire_nom"},
"minWidth": "200px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
],
),
)
return table
@callback(
Output("download-acheteur-data", "data"),
Input("btn-download-acheteur-data", "n_clicks"),
State(component_id="acheteur_data", component_property="data"),
State(component_id="acheteur_nom", component_property="children"),
State(component_id="acheteur_year", component_property="value"),
prevent_initial_call=True,
)
def download_acheteur_data(
n_clicks,
data: [dict],
acheteur_nom: str,
annee: str,
):
df_to_download = pl.DataFrame(data)
def to_bytes(buffer):
df_to_download.write_excel(
buffer, worksheet="DECP" if annee in ["Toutes", None] else annee
)
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
return dcc.send_bytes(to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx")
src/pages/home.py
+80 -125
View File
@@ -6,44 +6,18 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register
from dotenv import load_dotenv
from src.utils import (
add_annuaire_link,
add_org_links,
add_resource_link,
booleans_to_strings,
filter_table_data,
format_number,
lf,
logger,
split_filter_part,
sort_table_data,
)
load_dotenv()
update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH"))
update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y")
df_filtered = pl.DataFrame()
# Unique les données actuelles, pas les anciennes versions de marchés
lf = lf.filter(pl.col("donneesActuelles"))
# Suppression des colonnes inutiles
lf = lf.drop(
[
"donneesActuelles",
]
)
# Convertir les colonnes booléennes en chaînes de caractères
lf = booleans_to_strings(lf)
# Remplacer les valeurs manquantes par des chaînes vides
lf = lf.fill_null("")
# Ajout des liens vers l'annuaire
lf = add_annuaire_link(lf)
# Ajout des liens open data
lf = add_resource_link(lf)
schema = lf.collect_schema()
@@ -51,49 +25,41 @@ schema = lf.collect_schema()
title = "Tableau"
register_page(__name__, path="/", title="decp.info", name=title, order=1)
datatable = dash_table.DataTable(
cell_selectable=False,
id="table",
page_size=20,
page_current=0,
page_action="custom",
filter_action="custom",
filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."},
columns=[
{
"name": i,
"id": i,
"presentation": "markdown",
"type": "text",
"format": {"nully": "N/A"},
"hideable": True,
}
for i in lf.collect_schema().names()
],
sort_action="custom",
sort_mode="multi",
sort_by=[],
row_deletable=False,
style_cell_conditional=[
{
"if": {"column_id": "objet"},
"minWidth": "350px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
{
"if": {"column_id": "acheteur_nom"},
"minWidth": "250px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
],
data_timestamp=0,
markdown_options={"html": True},
datatable = html.Div(
className="marches_table",
children=dash_table.DataTable(
cell_selectable=False,
id="table",
page_size=20,
page_current=0,
page_action="custom",
filter_action="custom",
filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."},
sort_action="custom",
sort_mode="multi",
sort_by=[],
row_deletable=False,
style_cell_conditional=[
{
"if": {"column_id": "objet"},
"minWidth": "350px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
{
"if": {"column_id": "acheteur_nom"},
"minWidth": "250px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
],
data_timestamp=0,
markdown_options={"html": True},
),
)
layout = [
@@ -147,6 +113,7 @@ layout = [
disabled=True,
),
dcc.Download(id="download-data"),
dcc.Store(id="filtered_data", storage_type="memory"),
html.P("Données mises à jour le " + str(update_date)),
],
className="table-menu",
@@ -159,6 +126,8 @@ layout = [
@callback(
Output("table", "data"),
Output("table", "columns"),
# Output("filtered_data", "data"),
Output("table", "data_timestamp"),
Output("nb_rows", "children"),
Output("btn-download-data", "disabled"),
@@ -172,56 +141,21 @@ layout = [
)
def update_table(page_current, page_size, filter_query, sort_by, data_timestamp):
print(" + + + + + + + + + + + + + + + + + + ")
global df_filtered
# Application des filtres
lff: pl.LazyFrame = lf # start from the original data
if filter_query:
filtering_expressions = filter_query.split(" && ")
for filter_part in filtering_expressions:
col_name, operator, filter_value = split_filter_part(filter_part)
col_type = str(schema[col_name])
print("filter_value:", filter_value)
print("filter_value_type:", type(filter_value))
print("col_type:", col_type)
if operator in ("<", "<=", ">", ">="):
filter_value = int(filter_value)
if operator == "<":
lff = lff.filter(pl.col(col_name) < filter_value)
elif operator == ">":
lff = lff.filter(pl.col(col_name) > filter_value)
elif operator == ">=":
lff = lff.filter(pl.col(col_name) >= filter_value)
elif operator == "<=":
lff = lff.filter(pl.col(col_name) <= filter_value)
elif col_type.startswith("Int") or col_type.startswith("Float"):
try:
filter_value = int(filter_value)
except ValueError:
logger.error(f"Invalid numeric filter value: {filter_value}")
continue
lff = lff.filter(pl.col(col_name) == filter_value)
elif operator == "contains" and col_type == "String":
lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value))
# elif operator == 'datestartswith':
# lff = lff.filter(pl.col(col_name).str.startswith(filter_value)")
lff = filter_table_data(lff, filter_query)
if len(sort_by) > 0:
lff = lff.sort(
[col["column_id"] for col in sort_by],
descending=[col["direction"] == "desc" for col in sort_by],
nulls_last=True,
)
print(sort_by)
lff = sort_table_data(lff, sort_by)
# Remplace les strings null par "", mais pas les numeric null
lff = lff.fill_null("")
# Matérialisation des filtres
dff: pl.DataFrame = lff.collect()
df_filtered = dff.clone()
height = dff.height
nb_rows = f"{format_number(height)} lignes"
@@ -230,6 +164,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp)
start_row = page_current * page_size
# end_row = (page_current + 1) * page_size
dff = dff.slice(start_row, page_size)
# Ajout des liens vers l'annuaire des entreprises
dff = add_org_links(dff)
# Ajout des liens vers les fichiers Open Data
dff = add_resource_link(dff)
# Liste finale de colonnes
columns = [
{
"name": column,
"id": column,
"presentation": "markdown",
"type": "text",
"format": {"nully": "N/A"},
"hideable": True,
}
for column in dff.columns
]
dicts = dff.to_dicts()
if height > 65000:
@@ -243,6 +197,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp)
return (
dicts,
columns,
data_timestamp + 1,
nb_rows,
download_disabled,
@@ -254,26 +209,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp)
@callback(
Output("download-data", "data"),
Input("btn-download-data", "n_clicks"),
State("table", "filter_query"),
State("table", "sort_by"),
State("table", "hidden_columns"),
prevent_initial_call=True,
)
def download_data(n_clicks, hidden_columns: list = None):
df_to_download = df_filtered.clone()
print(df_to_download.columns)
# Rétablissement des colonnes source et sourceOpenData (voir add_resource_link)
df_to_download = df_to_download.with_columns(
pl.col("source").str.extract(r'href="(.*?)"').alias("sourceFile"),
pl.col("source").str.extract(r'">(.*?)<').alias("sourceDataset"),
)
def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None):
lff: pl.LazyFrame = lf # start from the original data
# Les colonnes masquées sont supprimées
if hidden_columns:
df_to_download = df_to_download.drop(hidden_columns)
lff = lff.drop(hidden_columns)
if filter_query:
lff = filter_table_data(lff, filter_query)
if len(sort_by) > 0:
lff = sort_table_data(lff, sort_by)
def to_bytes(buffer):
df_to_download.write_excel(buffer, worksheet="DECP")
lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
return dcc.send_bytes(to_bytes, filename=f"decp_{date}.xlsx")
src/utils.py
+94 -14
View File
@@ -1,3 +1,4 @@
import json
import logging
import os
from time import sleep
@@ -5,6 +6,7 @@ from time import sleep
import polars as pl
import polars.selectors as cs
from dotenv import load_dotenv
from httpx import get
from polars.exceptions import ComputeError
load_dotenv()
@@ -40,18 +42,18 @@ def split_filter_part(filter_part):
return [None] * 3
def add_resource_link(lff: pl.LazyFrame) -> pl.LazyFrame:
lff = lff.with_columns(
def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame:
dff = dff.with_columns(
(
'<a href="' + pl.col("sourceFile") + '">' + pl.col("sourceDataset") + "</a>"
).alias("source")
)
lff = lff.drop(["sourceFile", "sourceDataset"])
return lff
dff = dff.drop(["sourceFile", "sourceDataset"])
return dff
def add_annuaire_link(lff: pl.LazyFrame):
lff = lff.with_columns(
def add_org_links(dff: pl.DataFrame):
dff = dff.with_columns(
pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET")
.then(
'<a href = "https://annuaire-entreprises.data.gouv.fr/etablissement/'
@@ -63,16 +65,16 @@ def add_annuaire_link(lff: pl.LazyFrame):
.otherwise(pl.col("titulaire_id"))
.alias("titulaire_id")
)
lff = lff.with_columns(
dff = dff.with_columns(
(
'<a href = "https://annuaire-entreprises.data.gouv.fr/etablissement/'
'<a href = "/acheteur/'
+ pl.col("acheteur_id")
+ '">'
+ '" target="_blank">'
+ pl.col("acheteur_id")
+ "</a>"
).alias("acheteur_id")
)
return lff
return dff
def booleans_to_strings(lff: pl.LazyFrame) -> pl.LazyFrame:
@@ -101,6 +103,12 @@ def format_number(number) -> str:
return number
def get_annuaire_data(siret: str) -> dict:
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
response = get(url)
return response.json()["results"][0]
def get_decp_data() -> pl.LazyFrame:
# Chargement du fichier parquet
# Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment.
@@ -117,13 +125,85 @@ def get_decp_data() -> pl.LazyFrame:
sleep(10)
lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH"))
# Remplacement des valeurs numériques par des chaînes de caractères
# lff = numbers_to_strings(lff)
# Tri des marchés par date de notification
lff = lff.sort(by=["datePublicationDonnees"], descending=True, nulls_last=True)
lff = lff.sort(by=["dateNotification", "uid"], descending=True, nulls_last=True)
# Uniquement les données actuelles, pas les anciennes versions de marchés
lff = lff.filter(pl.col("donneesActuelles")).drop("donneesActuelles")
# Convertir les colonnes booléennes en chaînes de caractères
lff = booleans_to_strings(lff)
# Bizarrement je ne peux pas faire lff = lff.fill_null("") ici
# ça génère une erreur dans la page acheteur (acheteur_data.table) :
# AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import)
return lff
def get_departements() -> dict:
with open("data/departements.json", "rb") as f:
data = json.load(f)
return data
def get_departement_region(code_postal):
if code_postal > "97000":
code_departement = code_postal[:3]
else:
code_departement = code_postal[:2]
nom_departement = departements[code_departement]["departement"]
nom_region = departements[code_departement]["region"]
return code_departement, nom_departement, nom_region
def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame:
schema = lff.collect_schema()
filtering_expressions = filter_query.split(" && ")
for filter_part in filtering_expressions:
col_name, operator, filter_value = split_filter_part(filter_part)
col_type = str(schema[col_name])
print("filter_value:", filter_value)
print("filter_value_type:", type(filter_value))
print("col_type:", col_type)
if operator in ("<", "<=", ">", ">="):
filter_value = int(filter_value)
if operator == "<":
lff = lff.filter(pl.col(col_name) < filter_value)
elif operator == ">":
lff = lff.filter(pl.col(col_name) > filter_value)
elif operator == ">=":
lff = lff.filter(pl.col(col_name) >= filter_value)
elif operator == "<=":
lff = lff.filter(pl.col(col_name) <= filter_value)
elif col_type.startswith("Int") or col_type.startswith("Float"):
try:
filter_value = int(filter_value)
except ValueError:
logger.error(f"Invalid numeric filter value: {filter_value}")
continue
lff = lff.filter(pl.col(col_name) == filter_value)
elif operator == "contains" and col_type == "String":
lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value))
# elif operator == 'datestartswith':
# lff = lff.filter(pl.col(col_name).str.startswith(filter_value)")
return lff
def sort_table_data(lff: pl.LazyFrame, sort_by: list) -> pl.LazyFrame:
lff = lff.sort(
[col["column_id"] for col in sort_by],
descending=[col["direction"] == "desc" for col in sort_by],
nulls_last=True,
)
print(sort_by)
return lff
lf = get_decp_data()
departements = get_departements()