diff --git a/README.md b/README.md index 7092317..70a2fae 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,12 @@ python run.py ## Notes de version -#### 2.0.1 (23 septembre 2025) +#### 2.1.0 + +- Ajout de la vue acheteur +- Variables globales uniquement en lecture + +##### 2.0.1 (23 septembre 2025) - Bloquage du bouton de téléchargement si trop de lignes (+ 65000) [#38](https://github.com/ColinMaudry/decp.info/issues/38) - Amélioration du script de déploiement (deploy.sh) diff --git a/src/app.py b/src/app.py index e45d9ef..66e5f30 100644 --- a/src/app.py +++ b/src/app.py @@ -74,6 +74,7 @@ app.layout = html.Div( page["name"], href=page["relative_path"], className="nav" ) for page in page_registry.values() + if page["name"] not in ["Acheteur", "Fournisseur"] ] ), ], diff --git a/src/assets/style.css b/src/assets/style.css index 2532b92..873998a 100644 --- a/src/assets/style.css +++ b/src/assets/style.css @@ -71,8 +71,12 @@ td[data-dash-column="objet"] { } /* Alternance des couleurs pour les lignes */ -#table tr:nth-child(even) td { +.marches_table { + font-family: sans-serif; +} +.marches_table .cell-table tr:nth-child(even) td { background-color: #feeeee; + font-family: sans-serif; } #header > *, @@ -102,9 +106,51 @@ a.nav { } h3 { + margin: 36px 0 24px 0; +} + +summary > h3 { + margin: 0; display: inline; } #_pages_content { padding-top: 28px; } + +/* Vue acheteur/fournisseur */ +.wrapper { + display: grid; + grid-gap: 10px; + margin-bottom: 50px; + justify-content: space-between; +} + +.org_title { + grid-column: 1 / 3; + grid-row: 1; +} + +.org_year { + grid-column: 3; + grid-row: 1; +} + +.org_infos { + grid-column: 1; + grid-row: 2; +} + +.org_infos > p { + margin: 8px 0; +} + +.org_stats { + grid-column: 2; + grid-row: 2; +} + +.org_map { + grid-column: 3; + grid-row: 2; +} diff --git a/src/figures.py b/src/figures.py index 8ca0c5d..860c695 100644 --- a/src/figures.py +++ b/src/figures.py @@ -2,7 +2,7 @@ import json import plotly.express as px import polars as pl -from dash import dash_table, html +from dash import dash_table, dcc, html def get_map_count_marches(lf: pl.LazyFrame): @@ -157,3 +157,36 @@ def get_sources_tables(source_path) -> html.Div: datatable.data = df.to_dicts() return html.Div(children=datatable) + + +def point_on_map(lat, lon): + lat = float(lat) + lon = float(lon) + + # Create a scatter mapbox or choropleth map + fig = px.scatter_map( + lat=[lat], lon=[lon], height=300, width=400, color=[1], size=[1] + ) + + fig.update_coloraxes(showscale=False) + + # Set map style (you can use 'open-street-map', 'carto-positron', etc.) + fig.update_layout( + mapbox_style="light", # Light, clean background + margin={"r": 0, "t": 0, "l": 0, "b": 0}, + ) + + # Optionally, center the map on France + fig.update_geos( + center=dict(lat=46.603354, lon=1.888334), # Center of France + lataxis_range=[41, 51.5], # Latitude range for France + lonaxis_range=[-5, 10], # Longitude range for France + ) + + # But scatter_mapbox doesn't use geos, so better to control via zoom/center manually + # Let's reset and use proper centering in scatter_mapbox instead: + + fig.update_layout(map_center={"lat": 46.6, "lon": 1.89}, map_zoom=4) + + graph = dcc.Graph(id="map", figure=fig) + return graph diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py new file mode 100644 index 0000000..e3fa979 --- /dev/null +++ b/src/pages/acheteur.py @@ -0,0 +1,268 @@ +import datetime + +import polars as pl +from dash import Input, Output, State, callback, dash_table, dcc, html, register_page + +from src.figures import point_on_map +from src.utils import format_number, get_annuaire_data, get_departement_region, lf + +register_page( + __name__, + path_template="/acheteur/", + title="decp.info - acheteur", + name="Acheteur", + order=5, +) + +# 21690123100011 + +layout = [ + dcc.Store(id="acheteur_data", storage_type="memory"), + dcc.Location(id="url", refresh="callback-nav"), + html.Div( + className="container", + children=[ + html.Div( + className="wrapper", + children=[ + html.H2( + className="org_title", + children=[ + html.Span(id="acheteur_siret"), + " - ", + html.Span(id="acheteur_nom"), + ], + ), + html.Div( + className="org_year", + children=dcc.Dropdown( + id="acheteur_year", + options=["Toutes"] + + [ + str(year) + for year in range( + 2018, int(datetime.date.today().year) + 1 + ) + ], + placeholder="Année", + ), + ), + html.Div( + className="org_infos", + children=[ + # TODO: ajouter le type d'acheteur : commune, CD, CR, etc. + html.P(["Commune : ", html.Strong(id="acheteur_commune")]), + html.P( + [ + "Département : ", + html.Strong(id="acheteur_departement"), + ] + ), + html.P(["Région : ", html.Strong(id="acheteur_region")]), + html.A( + id="acheteur_lien_annuaire", + children="Plus de détails sur l'Annuaire des entreprises", + target="_blank", + ), + ], + ), + html.Div( + className="org_stats", + children=[ + html.P(id="acheteur_titre_stats"), + html.P(id="acheteur_marches_attribues"), + html.P(id="acheteur_fournisseurs_differents"), + html.Button( + "Téléchargement au format Excel", + id="btn-download-acheteur-data", + ), + dcc.Download(id="download-acheteur-data"), + ], + ), + html.Div(className="org_map", id="acheteur_map"), + ], + ), + # récupérer les données de l'acheteur sur l'api annuaire + html.H3("Derniers marchés publics attribués"), + html.Div(id="acheteur_last_marches", children=""), + ], + ), +] + + +@callback( + Output(component_id="acheteur_siret", component_property="children"), + Output(component_id="acheteur_nom", component_property="children"), + Output(component_id="acheteur_commune", component_property="children"), + Output(component_id="acheteur_map", component_property="children"), + Output(component_id="acheteur_departement", component_property="children"), + Output(component_id="acheteur_region", component_property="children"), + Output(component_id="acheteur_lien_annuaire", component_property="href"), + Input(component_id="url", component_property="pathname"), +) +def update_acheteur_infos(url): + acheteur_siret = url.split("/")[-1] + if len(acheteur_siret) != 14: + acheteur_siret = ( + f"Le SIRET renseigné doit faire 14 caractères ({acheteur_siret})" + ) + data = get_annuaire_data(acheteur_siret) + data_etablissement = data["matching_etablissements"][0] + acheteur_map = point_on_map( + data_etablissement["latitude"], data_etablissement["longitude"] + ) + code_departement, nom_departement, nom_region = get_departement_region( + data_etablissement["code_postal"] + ) + departement = f"{nom_departement} ({code_departement})" + lien_annuaire = ( + f"https://annuaire-entreprises.data.gouv.fr/etablissement/{acheteur_siret}" + ) + return ( + acheteur_siret, + data["nom_raison_sociale"], + data_etablissement["libelle_commune"], + acheteur_map, + departement, + nom_region, + lien_annuaire, + ) + + +@callback( + Output(component_id="acheteur_marches_attribues", component_property="children"), + Output( + component_id="acheteur_fournisseurs_differents", component_property="children" + ), + Input(component_id="acheteur_data", component_property="data"), +) +def update_acheteur_stats(data): + df = pl.DataFrame(data) + if df.height == 0: + df = pl.DataFrame(schema=lf.collect_schema()) + df_marches = df.unique("id") + nb_marches = format_number(df_marches.height) + # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) + marches_attribues = [html.Strong(nb_marches), " marchés et accord-cadres attribués"] + # + ", pour un total de ", html.Strong(somme_marches + " €")] + del df_marches + + nb_fournisseurs = df.unique("titulaire_id").height + nb_fournisseurs = [ + html.Strong(format_number(nb_fournisseurs)), + " fournisseurs (SIRET) différents", + ] + del df + + return marches_attribues, nb_fournisseurs + + +@callback( + Output(component_id="acheteur_data", component_property="data"), + Input(component_id="url", component_property="pathname"), + Input(component_id="acheteur_year", component_property="value"), +) +def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: + acheteur_siret = url.split("/")[-1] + lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + lff = lff.fill_null("") + lff = lff.select( + "id", + "objet", + "dateNotification", + "titulaire_id", + "titulaire_nom", + "montant", + "codeCPV", + "dureeMois", + ) + if acheteur_year and acheteur_year != "Toutes": + lff = lff.filter( + pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year) + ) + lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True) + + data = lff.collect(engine="streaming").to_dicts() + return data + + +@callback( + Output(component_id="acheteur_last_marches", component_property="children"), + Input(component_id="acheteur_data", component_property="data"), +) +def get_last_marches_table(data) -> html.Div: + columns = [ + "id", + "objet", + "dateNotification", + "titulaire_nom", + "montant", + "codeCPV", + "dureeMois", + ] + + table = html.Div( + className="marches_table", + id="marches_datatable", + children=dash_table.DataTable( + data=data, + page_action="native", + columns=[ + { + "name": i, + "id": i, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": False, + } + for i in columns + if i not in ["titulaire_id"] + ], + page_size=10, + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "300px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "titulaire_nom"}, + "minWidth": "200px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + ), + ) + return table + + +@callback( + Output("download-acheteur-data", "data"), + Input("btn-download-acheteur-data", "n_clicks"), + State(component_id="acheteur_data", component_property="data"), + State(component_id="acheteur_nom", component_property="children"), + State(component_id="acheteur_year", component_property="value"), + prevent_initial_call=True, +) +def download_acheteur_data( + n_clicks, + data: [dict], + acheteur_nom: str, + annee: str, +): + df_to_download = pl.DataFrame(data) + + def to_bytes(buffer): + df_to_download.write_excel( + buffer, worksheet="DECP" if annee in ["Toutes", None] else annee + ) + + date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + return dcc.send_bytes(to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx") diff --git a/src/pages/home.py b/src/pages/home.py index a33fae3..9e19d8c 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -6,44 +6,18 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from dotenv import load_dotenv from src.utils import ( - add_annuaire_link, + add_org_links, add_resource_link, - booleans_to_strings, + filter_table_data, format_number, lf, - logger, - split_filter_part, + sort_table_data, ) load_dotenv() update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -df_filtered = pl.DataFrame() - -# Unique les données actuelles, pas les anciennes versions de marchés - -lf = lf.filter(pl.col("donneesActuelles")) - -# Suppression des colonnes inutiles -lf = lf.drop( - [ - "donneesActuelles", - ] -) - -# Convertir les colonnes booléennes en chaînes de caractères -lf = booleans_to_strings(lf) - -# Remplacer les valeurs manquantes par des chaînes vides -lf = lf.fill_null("") - - -# Ajout des liens vers l'annuaire -lf = add_annuaire_link(lf) - -# Ajout des liens open data -lf = add_resource_link(lf) schema = lf.collect_schema() @@ -51,49 +25,41 @@ schema = lf.collect_schema() title = "Tableau" register_page(__name__, path="/", title="decp.info", name=title, order=1) -datatable = dash_table.DataTable( - cell_selectable=False, - id="table", - page_size=20, - page_current=0, - page_action="custom", - filter_action="custom", - filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, - columns=[ - { - "name": i, - "id": i, - "presentation": "markdown", - "type": "text", - "format": {"nully": "N/A"}, - "hideable": True, - } - for i in lf.collect_schema().names() - ], - sort_action="custom", - sort_mode="multi", - sort_by=[], - row_deletable=False, - style_cell_conditional=[ - { - "if": {"column_id": "objet"}, - "minWidth": "350px", - "textAlign": "left", - "overflow": "hidden", - "lineHeight": "14px", - "whiteSpace": "normal", - }, - { - "if": {"column_id": "acheteur_nom"}, - "minWidth": "250px", - "textAlign": "left", - "overflow": "hidden", - "lineHeight": "14px", - "whiteSpace": "normal", - }, - ], - data_timestamp=0, - markdown_options={"html": True}, +datatable = html.Div( + className="marches_table", + children=dash_table.DataTable( + cell_selectable=False, + id="table", + page_size=20, + page_current=0, + page_action="custom", + filter_action="custom", + filter_options={"case": "insensitive", "placeholder_text": "Filtrer..."}, + sort_action="custom", + sort_mode="multi", + sort_by=[], + row_deletable=False, + style_cell_conditional=[ + { + "if": {"column_id": "objet"}, + "minWidth": "350px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + { + "if": {"column_id": "acheteur_nom"}, + "minWidth": "250px", + "textAlign": "left", + "overflow": "hidden", + "lineHeight": "14px", + "whiteSpace": "normal", + }, + ], + data_timestamp=0, + markdown_options={"html": True}, + ), ) layout = [ @@ -147,6 +113,7 @@ layout = [ disabled=True, ), dcc.Download(id="download-data"), + dcc.Store(id="filtered_data", storage_type="memory"), html.P("Données mises à jour le " + str(update_date)), ], className="table-menu", @@ -159,6 +126,8 @@ layout = [ @callback( Output("table", "data"), + Output("table", "columns"), + # Output("filtered_data", "data"), Output("table", "data_timestamp"), Output("nb_rows", "children"), Output("btn-download-data", "disabled"), @@ -172,56 +141,21 @@ layout = [ ) def update_table(page_current, page_size, filter_query, sort_by, data_timestamp): print(" + + + + + + + + + + + + + + + + + + ") - global df_filtered # Application des filtres lff: pl.LazyFrame = lf # start from the original data if filter_query: - filtering_expressions = filter_query.split(" && ") - for filter_part in filtering_expressions: - col_name, operator, filter_value = split_filter_part(filter_part) - col_type = str(schema[col_name]) - print("filter_value:", filter_value) - print("filter_value_type:", type(filter_value)) - print("col_type:", col_type) - - if operator in ("<", "<=", ">", ">="): - filter_value = int(filter_value) - if operator == "<": - lff = lff.filter(pl.col(col_name) < filter_value) - elif operator == ">": - lff = lff.filter(pl.col(col_name) > filter_value) - elif operator == ">=": - lff = lff.filter(pl.col(col_name) >= filter_value) - elif operator == "<=": - lff = lff.filter(pl.col(col_name) <= filter_value) - - elif col_type.startswith("Int") or col_type.startswith("Float"): - try: - filter_value = int(filter_value) - except ValueError: - logger.error(f"Invalid numeric filter value: {filter_value}") - continue - lff = lff.filter(pl.col(col_name) == filter_value) - - elif operator == "contains" and col_type == "String": - lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) - - # elif operator == 'datestartswith': - # lff = lff.filter(pl.col(col_name).str.startswith(filter_value)") + lff = filter_table_data(lff, filter_query) if len(sort_by) > 0: - lff = lff.sort( - [col["column_id"] for col in sort_by], - descending=[col["direction"] == "desc" for col in sort_by], - nulls_last=True, - ) - print(sort_by) + lff = sort_table_data(lff, sort_by) + # Remplace les strings null par "", mais pas les numeric null + lff = lff.fill_null("") + + # Matérialisation des filtres dff: pl.DataFrame = lff.collect() - df_filtered = dff.clone() - height = dff.height nb_rows = f"{format_number(height)} lignes" @@ -230,6 +164,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) start_row = page_current * page_size # end_row = (page_current + 1) * page_size dff = dff.slice(start_row, page_size) + + # Ajout des liens vers l'annuaire des entreprises + dff = add_org_links(dff) + + # Ajout des liens vers les fichiers Open Data + dff = add_resource_link(dff) + + # Liste finale de colonnes + columns = [ + { + "name": column, + "id": column, + "presentation": "markdown", + "type": "text", + "format": {"nully": "N/A"}, + "hideable": True, + } + for column in dff.columns + ] + dicts = dff.to_dicts() if height > 65000: @@ -243,6 +197,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) return ( dicts, + columns, data_timestamp + 1, nb_rows, download_disabled, @@ -254,26 +209,26 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) @callback( Output("download-data", "data"), Input("btn-download-data", "n_clicks"), + State("table", "filter_query"), + State("table", "sort_by"), State("table", "hidden_columns"), prevent_initial_call=True, ) -def download_data(n_clicks, hidden_columns: list = None): - df_to_download = df_filtered.clone() - - print(df_to_download.columns) - - # Rétablissement des colonnes source et sourceOpenData (voir add_resource_link) - df_to_download = df_to_download.with_columns( - pl.col("source").str.extract(r'href="(.*?)"').alias("sourceFile"), - pl.col("source").str.extract(r'">(.*?)<').alias("sourceDataset"), - ) +def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None): + lff: pl.LazyFrame = lf # start from the original data # Les colonnes masquées sont supprimées if hidden_columns: - df_to_download = df_to_download.drop(hidden_columns) + lff = lff.drop(hidden_columns) + + if filter_query: + lff = filter_table_data(lff, filter_query) + + if len(sort_by) > 0: + lff = sort_table_data(lff, sort_by) def to_bytes(buffer): - df_to_download.write_excel(buffer, worksheet="DECP") + lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP") date = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") return dcc.send_bytes(to_bytes, filename=f"decp_{date}.xlsx") diff --git a/src/utils.py b/src/utils.py index 3c6856e..96c2bfb 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,3 +1,4 @@ +import json import logging import os from time import sleep @@ -5,6 +6,7 @@ from time import sleep import polars as pl import polars.selectors as cs from dotenv import load_dotenv +from httpx import get from polars.exceptions import ComputeError load_dotenv() @@ -40,18 +42,18 @@ def split_filter_part(filter_part): return [None] * 3 -def add_resource_link(lff: pl.LazyFrame) -> pl.LazyFrame: - lff = lff.with_columns( +def add_resource_link(dff: pl.DataFrame) -> pl.DataFrame: + dff = dff.with_columns( ( '' + pl.col("sourceDataset") + "" ).alias("source") ) - lff = lff.drop(["sourceFile", "sourceDataset"]) - return lff + dff = dff.drop(["sourceFile", "sourceDataset"]) + return dff -def add_annuaire_link(lff: pl.LazyFrame): - lff = lff.with_columns( +def add_org_links(dff: pl.DataFrame): + dff = dff.with_columns( pl.when(pl.col("titulaire_typeIdentifiant") == "SIRET") .then( '' + + '" target="_blank">' + pl.col("acheteur_id") + "" ).alias("acheteur_id") ) - return lff + return dff def booleans_to_strings(lff: pl.LazyFrame) -> pl.LazyFrame: @@ -101,6 +103,12 @@ def format_number(number) -> str: return number +def get_annuaire_data(siret: str) -> dict: + url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" + response = get(url) + return response.json()["results"][0] + + def get_decp_data() -> pl.LazyFrame: # Chargement du fichier parquet # Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment. @@ -117,13 +125,85 @@ def get_decp_data() -> pl.LazyFrame: sleep(10) lff: pl.LazyFrame = pl.scan_parquet(os.getenv("DATA_FILE_PARQUET_PATH")) - # Remplacement des valeurs numériques par des chaînes de caractères - # lff = numbers_to_strings(lff) - # Tri des marchés par date de notification - lff = lff.sort(by=["datePublicationDonnees"], descending=True, nulls_last=True) + lff = lff.sort(by=["dateNotification", "uid"], descending=True, nulls_last=True) + + # Uniquement les données actuelles, pas les anciennes versions de marchés + lff = lff.filter(pl.col("donneesActuelles")).drop("donneesActuelles") + + # Convertir les colonnes booléennes en chaînes de caractères + lff = booleans_to_strings(lff) + + # Bizarrement je ne peux pas faire lff = lff.fill_null("") ici + # ça génère une erreur dans la page acheteur (acheteur_data.table) : + # AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import) return lff +def get_departements() -> dict: + with open("data/departements.json", "rb") as f: + data = json.load(f) + return data + + +def get_departement_region(code_postal): + if code_postal > "97000": + code_departement = code_postal[:3] + else: + code_departement = code_postal[:2] + nom_departement = departements[code_departement]["departement"] + nom_region = departements[code_departement]["region"] + return code_departement, nom_departement, nom_region + + +def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: + schema = lff.collect_schema() + filtering_expressions = filter_query.split(" && ") + for filter_part in filtering_expressions: + col_name, operator, filter_value = split_filter_part(filter_part) + col_type = str(schema[col_name]) + print("filter_value:", filter_value) + print("filter_value_type:", type(filter_value)) + print("col_type:", col_type) + + if operator in ("<", "<=", ">", ">="): + filter_value = int(filter_value) + if operator == "<": + lff = lff.filter(pl.col(col_name) < filter_value) + elif operator == ">": + lff = lff.filter(pl.col(col_name) > filter_value) + elif operator == ">=": + lff = lff.filter(pl.col(col_name) >= filter_value) + elif operator == "<=": + lff = lff.filter(pl.col(col_name) <= filter_value) + + elif col_type.startswith("Int") or col_type.startswith("Float"): + try: + filter_value = int(filter_value) + except ValueError: + logger.error(f"Invalid numeric filter value: {filter_value}") + continue + lff = lff.filter(pl.col(col_name) == filter_value) + + elif operator == "contains" and col_type == "String": + lff = lff.filter(pl.col(col_name).str.contains("(?i)" + filter_value)) + + # elif operator == 'datestartswith': + # lff = lff.filter(pl.col(col_name).str.startswith(filter_value)") + + return lff + + +def sort_table_data(lff: pl.LazyFrame, sort_by: list) -> pl.LazyFrame: + lff = lff.sort( + [col["column_id"] for col in sort_by], + descending=[col["direction"] == "desc" for col in sort_by], + nulls_last=True, + ) + print(sort_by) + return lff + + lf = get_decp_data() +departements = get_departements()