from typing import Literal
from urllib.error import HTTPError, URLError
import dash_bootstrap_components as dbc
import dash_leaflet as dl
import dash_leaflet.express as dlx
import plotly.express as px
import plotly.graph_objects as go
import polars as pl
from dash import dash_table, dcc, html
from dash_extensions.javascript import Namespace
from src.utils import data_schema, departements_geojson, df, format_number
def get_yearly_statistics(statistics, today_str) -> html.Div:
# Build DataFrame from statistics
years = list(reversed(range(2018, int(today_str.split("/")[-1]) + 1)))
data = []
for year in years:
year_str = str(year)
stat = statistics[year_str]
data.append(
{
"Année": year_str,
"Marchés et accord-cadres": format_number(
stat["nb_notifications_marches"]
),
"Acheteurs": format_number(stat["nb_acheteurs_uniques"]),
"Titulaires": format_number(stat["nb_titulaires_uniques"]),
}
)
dff = pl.DataFrame(data)
# Create Dash DataTable
table = dash_table.DataTable(
data=dff.to_dicts(),
columns=[
{"name": "Année", "id": "Année"},
{"name": "Marchés et accord-cadres", "id": "Marchés et accord-cadres"},
{"name": "Acheteurs", "id": "Acheteurs"},
{"name": "Titulaires", "id": "Titulaires"},
],
page_size=10,
sort_action="none",
filter_action="none",
style_header={"fontFamily": "Inter", "fontSize": "16px"},
style_cell={"fontFamily": "Inter", "fontSize": "16px"},
)
return html.Div(children=table, className="marches_table")
def get_barchart_sources(df_source: pl.DataFrame, type_date: str):
lf = df_source.lazy()
labels = {
"dateNotification": "notification",
"datePublicationDonnees": "publication des données",
}
lf = lf.select("uid", type_date, "sourceDataset")
lf = lf.unique("uid")
# Rassemblement des datasets Atexo pour ne pas surcharger le graphique
lf = lf.with_columns(
pl.when(pl.col("sourceDataset").str.starts_with("atexo"))
.then(pl.lit("plateformes atexo"))
.otherwise(pl.col("sourceDataset"))
.alias("sourceDataset")
)
# Rassemblement des datasets AWS pour ne pas surcharger le graphique
lf = lf.with_columns(
pl.when(pl.col("sourceDataset").str.contains(r"aws|marches\-publics.info"))
.then(pl.lit("aws"))
.otherwise(pl.col("sourceDataset"))
.alias("sourceDataset")
)
lf = lf.with_columns(pl.col(type_date).dt.year().alias("annee"))
lf = lf.filter(
pl.col(type_date).is_not_null() & pl.col("annee").is_between(2019, 2025)
)
lf = lf.with_columns(pl.col(type_date).cast(pl.String).str.head(7))
lf = (
lf.group_by([type_date, "sourceDataset"])
.len()
.sort(by=[type_date, "len"], descending=True)
)
# lf = lf.with_columns(
# pl.when(pl.col("sourceDataset").is_null()).then(
# pl.lit("Source inconnue")).alias("sourceDataset")
# )
lf = lf.sort(by=["sourceDataset"], descending=False)
df: pl.DataFrame = lf.collect(engine="streaming")
fig = px.bar(
df,
x=type_date,
y="len",
color="sourceDataset",
title=f"Nombre de marchés attribués par date de {labels[type_date]} et source de données",
labels={
"len": "Nombre de marchés",
type_date: f"Mois de {labels[type_date]}",
"sourceDataset": "Source de données",
},
)
return fig
def get_sources_tables(source_path) -> html.Div:
try:
dff = pl.read_csv(source_path)
except (URLError, HTTPError):
return html.Div("Erreur de connexion")
dff = dff.with_columns(
(
pl.lit('')
+ pl.col("nom")
+ pl.lit("")
).alias("nom")
)
dff = dff.drop("url", "unique")
dff = dff.sort(by=["nb_marchés"], descending=True)
columns = {
"nom": "Nom de la source",
"organisation": "Responsable de publication",
"nb_marchés": "Nb de marchés",
"nb_acheteurs": "Nb d'acheteurs",
"code": "Code",
}
datatable = dash_table.DataTable(
id="source_table",
data=dff.to_dicts(),
columns=[
{
"name": columns[i],
"id": i,
"presentation": "markdown",
"type": "text",
"format": {"nully": "N/A"},
}
for i in dff.schema.names()
],
style_cell_conditional=[
{
"if": {"column_id": ["nom", "organisation"]},
"minWidth": "350px",
"textAlign": "left",
"overflow": "hidden",
"lineHeight": "14px",
"whiteSpace": "normal",
},
],
sort_action="native",
markdown_options={"html": True},
style_header={"fontFamily": "Inter", "fontSize": "16px"},
style_cell={"fontFamily": "Inter", "fontSize": "16px"},
)
return html.Div(children=datatable)
def point_on_map(lat, lon):
lat = float(lat)
lon = float(lon)
# Create a scatter mapbox or choropleth map
fig = px.scatter_map(
lat=[lat], lon=[lon], height=300, width=400, color=[1], size=[1]
)
fig.update_coloraxes(showscale=False)
# Set map style (you can use 'open-street-map', 'carto-positron', etc.)
fig.update_layout(
mapbox_style="light", # Light, clean background
margin={"r": 0, "t": 0, "l": 0, "b": 0},
)
# Optionally, center the map on France
fig.update_geos(
center=dict(lat=46.603354, lon=1.888334), # Center of France
lataxis_range=[41, 51.5], # Latitude range for France
lonaxis_range=[-5, 10], # Longitude range for France
)
# But scatter_mapbox doesn't use geos, so better to control via zoom/center manually
# Let's reset and use proper centering in scatter_mapbox instead:
fig.update_layout(map_center={"lat": 46.6, "lon": 1.89}, map_zoom=4)
graph = dcc.Graph(id="map", figure=fig)
graph = html.Div(style={"width": "400px"})
return graph
class DataTable(dash_table.DataTable):
def __init__(
self,
dtid: str,
hidden_columns: list[str] | None = None,
data: list[dict[str, str | int | float | bool]] | None = None,
columns: list[dict[str, str]] | None = None,
page_size: int = 20,
page_action: Literal["native", "custom", "none"] = "native",
sort_action: Literal["native", "custom", "none"] = "native",
filter_action: Literal["native", "custom", "none"] = "native",
style_cell_conditional: list | None = None,
style_cell: dict | None = None,
**kwargs,
):
# Styles de base
style_cell_conditional_common = [
{
"if": {"column_id": "objet"},
"minWidth": "350px",
"overflow": "hidden",
"lineHeight": "18px",
"whiteSpace": "normal",
},
{
"if": {"column_id": "acheteur_id"},
"minWidth": "160px",
"overflow": "hidden",
"whiteSpace": "normal",
},
{
"if": {"column_id": "acheteur_nom"},
"minWidth": "250px",
"overflow": "hidden",
"lineHeight": "18px",
"whiteSpace": "normal",
},
{
"if": {"column_id": "titulaire_nom"},
"minWidth": "250px",
"overflow": "hidden",
"lineHeight": "18px",
"whiteSpace": "normal",
},
]
style_cell_common = {"fontFamily": "Inter", "fontSize": "16px"}
for key in data_schema.keys():
field = data_schema[key]
if field["type"] in ["number", "integer"]:
rule = {
"if": {"column_id": field["name"]},
"textAlign": "right",
# "fontFamily": "Fira Code",
}
style_cell_conditional_common.append(rule)
style_cell_conditional = (
style_cell_conditional or []
) + style_cell_conditional_common
if style_cell:
style_cell.update(style_cell_common)
else:
style_cell = style_cell_common
style_header = style_cell
# Initialisation de la classe parente avec les arguments
super().__init__(
id=dtid,
data=data,
columns=columns,
cell_selectable=False,
page_size=page_size,
filter_action=filter_action,
page_action=page_action,
filter_options={
"case": "insensitive",
"placeholder_text": "Filtre de colonne...",
},
sort_action=sort_action,
sort_mode="multi",
row_deletable=False,
page_current=0,
style_cell_conditional=style_cell_conditional,
data_timestamp=0,
markdown_options={"html": True},
style_header=style_header,
style_cell=style_cell,
tooltip_duration=8000,
tooltip_delay=350,
hidden_columns=hidden_columns,
**kwargs, # Possibilité de remplacer des arguments
)
def get_duplicate_matrix() -> html.Div:
"""
Fonction développée avec l'aide de la LLM Euria d'Infomaniak.
:return:
"""
result_df = pl.read_parquet(
"https://www.data.gouv.fr/api/1/datasets/r/a545bf6c-8b24-46ed-b49f-a32bf02eaffa"
).sort("sourceDataset")
result_df = result_df.select(
["sourceDataset", "unique"] + sorted(result_df.columns[2:])
)
description = dcc.Markdown("""
Ce graphique illustre les doublons de marchés publics entre sources, c'est-à-dire la proportion de marchés publiés par plus d'une source. Il s'appuie sur les identifiants `uid` qui sont pour chaque marché la concaténation du SIRET de l'acheteur et de l'identifiant interne du marché.
**Comment lire ce graphique ?**
On part des codes de sources de données en ordonnée. Ces jeux de données sont documentés dans [À propos](/a-propos#sources).
La première colonne (**unique**) représente le pourcentage de marchés fournis par cette source qui sont uniquement disponibles dans cette source. Plus le rouge est foncé, plus important est le pourcentage. Donc, à l'inverse, plus le rouge est clair dans la première colonne, plus la source en ordonnée a des marchés en commun avec d'autres sources, et donc plus on trouvera sur la même ligne d'autres cases plus ou moins foncées qui indiqueront avec quelles autres sources cette source partage des marchés.
Passez votre souris sur une case pour avoir les pourcentages exacts. À noter que ces statistiques sont produites avant le dédoublonnement qui a lieu avant la publication en Open Data et sur ce site.""")
# Extract data
z_data = result_df.select(pl.all().exclude("sourceDataset")).fill_null(0).to_numpy()
x_labels = result_df.columns[1:] # columns after "sourceDataset"
y_labels = result_df["sourceDataset"].to_list()
# Create heatmap
fig = go.Figure(
data=go.Heatmap(
z=z_data,
x=x_labels,
y=y_labels,
colorscale=[
[0.0, "white"], # 0% → white
[0.10, "lightsalmon"], # 10% → light warm tone
[1.0, "darkred"], # 100% → deep red
],
zmin=0,
zmax=1,
hoverongaps=False,
showscale=True,
hovertemplate=(
"%{z:.0%} des marchés de %{y} sont également présents dans %{x}"
),
)
)
# Update layout: make it wider and taller
fig.update_layout(
title="",
xaxis_title="Sources de données",
yaxis_title="Sources de données",
yaxis=dict(autorange="reversed"),
xaxis=dict(tickangle=45, tickfont=dict(size=10)), # Smaller x-tick labels
coloraxis_colorbar=dict(title="Percentage", tickfont=dict(size=10)),
width=1000, # Wider
height=1000, # Taller
font=dict(size=11), # Overall font size
margin=dict(l=100, r=50, t=80, b=100), # Add margin for labels
)
return html.Div(
children=[
html.H3("Doublons de marchés entre les sources"),
description,
dcc.Graph(figure=fig),
]
)
def get_geographic_maps(dff: pl.DataFrame) -> list | None:
"""
Génère les cartes géographiques pour la métropole et les DOM-TOM.
"""
regions: dict = {
"Métropole": {
"coordinates": [46.6, 2.2],
"zoom_leaflet": 5,
"zoom_chloropleth": 1,
"name": "Métropole",
},
"971": {
"coordinates": [16.23, -61.55],
"zoom_leaflet": 9,
"zoom_chloropleth": 1,
"name": "Guadeloupe",
},
"972": {
"coordinates": [14.64, -61.02],
"zoom_leaflet": 10,
"zoom_chloropleth": 1,
"name": "Martinique",
},
"973": {
"coordinates": [3.93, -53.12],
"zoom_leaflet": 7,
"zoom_chloropleth": 1,
"name": "Guyane",
},
"974": {
"coordinates": [-21.11, 55.53],
"zoom_leaflet": 9,
"zoom_chloropleth": 1,
"name": "La Réunion",
},
"976": {
"coordinates": [-12.82, 45.16],
"zoom_leaflet": 10,
"zoom_chloropleth": 1,
"name": "Mayotte",
},
}
def make_map_data(region_code: str) -> tuple[list, str or None]:
lff: pl.LazyFrame = dff.lazy()
if region_code == "Métropole":
lff = lff.filter(
(pl.col("acheteur_departement_code").str.len_chars() == 2)
& (pl.col("titulaire_departement_code").str.len_chars() == 2)
)
else:
lff = lff.filter(
(pl.col("acheteur_departement_code") == code)
| (pl.col("titulaire_departement_code") == code)
)
nb_marches = lff.select("uid").collect()["uid"].n_unique()
if nb_marches == 0:
return [], None
dfs = []
if (code == "Métropole" and nb_marches > 30000) or (
code != "Métropole" and nb_marches > 10000
):
_map_type: str = "chloropleth"
lff = lff.rename({"acheteur_departement_code": "Département"})
lff = (
lff.select(["uid", "Département"])
.drop_nulls()
.group_by("uid")
.agg(pl.col("Département").first())
.group_by("Département")
.len("uid")
)
dfs.append(lff.collect())
else:
_map_type: str = "clusters"
for org_type in ["acheteur", "titulaire"]:
lff_org = (
lff.select(
"uid",
f"{org_type}_longitude",
f"{org_type}_latitude",
f"{org_type}_nom",
)
.group_by(
f"{org_type}_longitude",
f"{org_type}_latitude",
f"{org_type}_nom",
)
.len("nb_marches")
.filter(
pl.col(f"{org_type}_latitude").is_not_null()
& pl.col(f"{org_type}_longitude").is_not_null()
)
)
markers = []
# Couleurs accessibles (Okabe-Ito)
colors = {
"acheteur": "#E69F00", # orange
"titulaire": "#56B4E9", # bleu ciel
}
for row in lff_org.collect().to_dicts():
markers.append(
{
"lat": row[f"{org_type}_latitude"],
"lon": row[f"{org_type}_longitude"],
"tooltip": f"{row[f'{org_type}_nom']} ({row['nb_marches']} marchés)",
"marker_color": colors[org_type],
}
)
dfs.append(markers)
return dfs, _map_type
cols = []
for code in regions.keys():
regions[code]["data"], map_type = make_map_data(code)
if map_type == "chloropleth":
map_graph = make_chloropleth_map(regions[code])
elif map_type == "clusters":
map_graph = make_clusters_map(regions[code])
elif map_type is None:
continue
else:
raise ValueError(f"Map type '{map_type}' not recognised")
lg, xl = (12, 8) if code == "Métropole" else (6, 4)
col = make_card(regions[code]["name"], fig=map_graph, lg=lg, xl=xl)
cols.append(col)
return cols
def make_chloropleth_map(region: dict) -> dcc.Graph:
df_map = region["data"][0]
fig = px.choropleth(
df_map,
geojson=departements_geojson,
locations="Département",
color="uid",
color_continuous_scale="Reds",
range_color=(df_map["uid"].min(), df_map["uid"].max()),
labels={"uid": "Marchés attribués"},
scope="europe",
)
fig.update_geos(fitbounds="locations", visible=False)
fig.update_layout(
mapbox={
"style": "carto-positron",
"center": {"lon": 10, "lat": 10},
"zoom": 8,
"domain": {"x": [0, 1], "y": [0, 1]},
}
)
graph = dcc.Graph(figure=fig, config={"displayModeBar": False})
return graph
def make_clusters_map(region: dict) -> dl.Map:
# JavaScript functions for styling
ns = Namespace("dash_clientside", "leaflet")
point_to_layer = ns("pointToLayer")
cluster_to_layer = ns("clusterToLayer")
name = region["name"]
# Données de la région
region_acheteurs = region["data"][0]
region_titulaires = region["data"][1]
# Couleurs
color_acheteur = region_acheteurs[0]["marker_color"]
color_titulaire = region_titulaires[0]["marker_color"]
acheteurs_geojson_data = dlx.dicts_to_geojson(region_acheteurs)
titulaires_geojson_data = dlx.dicts_to_geojson(region_titulaires)
center, zoom = region["coordinates"], region["zoom_leaflet"]
region_id = name.lower().replace(" ", "-")
leaflet_map = dl.Map(
[
dl.TileLayer(),
dl.GeoJSON(
data=titulaires_geojson_data,
cluster=True,
zoomToBoundsOnClick=True,
pointToLayer=point_to_layer,
clusterToLayer=cluster_to_layer,
id=f"geojson-{region_id}-titulaires",
options={"fillColor": color_titulaire},
),
dl.GeoJSON(
data=acheteurs_geojson_data,
cluster=True,
zoomToBoundsOnClick=True,
pointToLayer=point_to_layer,
clusterToLayer=cluster_to_layer,
id=f"geojson-{region_id}-acheteurs",
options={"fillColor": color_acheteur},
),
],
center=center,
zoom=zoom,
style={
"width": "100%",
"height": "400px" if name == "Métropole" else "300px",
},
id=f"map-{region_id}",
)
return leaflet_map
def make_card(
title: str, subtitle=None, fig=None, paragraphs=None, lg=6, xl=4
) -> dbc.Col:
children = []
if title:
children.append(html.H5(title, className="card-title"))
if subtitle:
children.append(html.H6(subtitle, className="card-subtitle mb-2 text-muted"))
if fig is not None:
children.append(fig)
if paragraphs:
for p in paragraphs:
p.className = "card-text"
children.append(p)
card = dbc.Col(
html.Div(html.Div(className="card-body", children=children), className="card"),
lg=lg,
xl=xl,
# width=width,
# className="mb-4",
)
return card
def make_donut(lff: pl.LazyFrame, names_col):
title = data_schema[names_col]["title"]
lff = lff.rename({names_col: title})
lff = lff.select("uid", title)
lff = lff.group_by(title).len("Nombre")
lff = lff.with_columns(pl.col(title).replace(None, pl.lit("?")))
fig = px.pie(
lff.collect(engine="streaming"),
values="Nombre",
names=title,
hole=0.4,
color_discrete_sequence=px.colors.qualitative.Safe,
)
fig = fig.update_traces(texttemplate="%{label}
%{percent}")
fig = fig.update_layout(showlegend=False, font=dict(size=14))
graph = dcc.Graph(figure=fig)
return graph
def make_column_picker(page: str):
table_data = []
table_columns = [
{
"id": col,
"name": data_schema[col]["title"],
"description": data_schema[col]["description"],
}
for col in df.columns
]
for column in table_columns:
new_column = {
"id": column["id"],
"name": column["name"],
"description": data_schema[column["id"]]["description"],
}
table_data.append(new_column)
table = (
DataTable(
row_selectable="multi",
data=table_data,
filter_action="native",
sort_action="none",
style_cell={
"textAlign": "left",
},
columns=[
{
"name": "Nom",
"id": "name",
},
{
"name": "Description",
"id": "description",
},
],
style_cell_conditional=[
{
"if": {"column_id": "description"},
"minWidth": "450px",
"overflow": "hidden",
"lineHeight": "18px",
"whiteSpace": "normal",
}
],
page_action="none",
dtid=f"{page}_column_list",
),
)
return table