colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

Global lf => global df pour être moins sensible au update de fichier source

Browse Source
This commit is contained in:
Colin Maudry
2025-10-03 16:29:58 +02:00
parent c2fab377d5
commit c7768ea8f1
7 changed files with 44 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/figures.py
+6 -4
View File
@@ -5,7 +5,8 @@ import polars as pl
from dash import dash_table, dcc, html
def get_map_count_marches(lf: pl.LazyFrame):
def get_map_count_marches(df: pl.DataFrame):
lf = df.lazy()
lf = lf.with_columns(
pl.col("lieuExecution_code").str.head(2).str.zfill(2).alias("Département")
)
@@ -26,7 +27,7 @@ def get_map_count_marches(lf: pl.LazyFrame):
for f in departements["features"]:
f["id"] = f["properties"]["code"]
df = lf.collect()
df = lf.collect(engine="streaming")
fig = px.choropleth(
df,
@@ -54,7 +55,8 @@ def get_map_count_marches(lf: pl.LazyFrame):
return fig
def get_barchart_sources(lf: pl.LazyFrame, type_date: str):
def get_barchart_sources(df: pl.DataFrame, type_date: str):
lf = df.lazy()
labels = {
"dateNotification": "notification",
"datePublicationDonnees": "publication des données",
@@ -97,7 +99,7 @@ def get_barchart_sources(lf: pl.LazyFrame, type_date: str):
# )
lf = lf.sort(by=["sourceDataset"], descending=False)
df: pl.DataFrame = lf.collect()
df: pl.DataFrame = lf.collect(engine="streaming")
fig = px.bar(
df,
src/pages/acheteur.py
+5 -4
View File
@@ -6,11 +6,11 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register
from src.figures import point_on_map
from src.utils import (
add_links_in_dict,
df,
format_montant,
format_number,
get_annuaire_data,
get_departement_region,
lf,
meta_content,
setup_table_columns,
)
@@ -148,7 +148,7 @@ def update_acheteur_infos(url):
def update_acheteur_stats(data):
df = pl.DataFrame(data)
if df.height == 0:
df = pl.DataFrame(schema=lf.collect_schema())
df = pl.DataFrame(schema=df.collect_schema())
df_marches = df.unique("id")
nb_marches = format_number(df_marches.height)
# somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item()))
@@ -171,9 +171,10 @@ def update_acheteur_stats(data):
Input(component_id="url", component_property="pathname"),
Input(component_id="acheteur_year", component_property="value"),
)
def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame:
def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]:
acheteur_siret = url.split("/")[-1]
lff = lf.filter(pl.col("acheteur_id") == acheteur_siret)
lff = df.lazy()
lff = lff.filter(pl.col("acheteur_id") == acheteur_siret)
lff = lff.fill_null("")
lff = lff.select(
"id",
src/pages/home.py
+6 -5
View File
@@ -7,10 +7,10 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register
from src.utils import (
add_links,
add_resource_link,
df,
filter_table_data,
format_montant,
format_number,
lf,
meta_content,
setup_table_columns,
sort_table_data,
@@ -19,7 +19,7 @@ from src.utils import (
update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH"))
update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y")
schema = lf.collect_schema()
schema = df.collect_schema()
name = "Tableau"
register_page(
@@ -171,10 +171,11 @@ layout = [
State("table", "data_timestamp"),
)
def update_table(page_current, page_size, filter_query, sort_by, data_timestamp):
print(" + + + + + + + + + + + + + + + + + + ")
if os.getenv("DEVELOPMENT").lower() == "true":
print(" + + + + + + + + + + + + + + + + + + ")
# Application des filtres
lff: pl.LazyFrame = lf # start from the original data
lff: pl.LazyFrame = df.lazy() # start from the original data
if filter_query:
lff = filter_table_data(lff, filter_query)
@@ -238,7 +239,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp)
prevent_initial_call=True,
)
def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None):
lff: pl.LazyFrame = lf # start from the original data
lff: pl.LazyFrame = df # start from the original data
# Les colonnes masquées sont supprimées
if hidden_columns:
src/pages/marche.py
+5 -4
View File
@@ -4,7 +4,7 @@ import polars as pl
from dash import Input, Output, callback, dcc, html, register_page
from polars import selectors as cs
from src.utils import data_schema, format_montant, lf, meta_content
from src.utils import data_schema, df, format_montant, meta_content
register_page(
__name__,
@@ -62,11 +62,12 @@ layout = [
Output("titulaires_data", "data"),
Input(component_id="url", component_property="pathname"),
)
def get_marche_data(url):
def get_marche_data(url) -> tuple[dict, list]:
marche_uid = url.split("/")[-1]
# Récupération des données du marché à partir du lf global
lff = lf.filter(pl.col("uid") == pl.lit(marche_uid))
# Récupération des données du marché à partir du df global
lff = df.lazy()
lff = lff.filter(pl.col("uid") == pl.lit(marche_uid))
# Données des titulaires du marché
dff_titulaires = lff.select(cs.starts_with("titulaire")).collect(engine="streaming")
src/pages/statistiques.py
+4 -4
View File
@@ -1,7 +1,7 @@
from dash import dcc, html, register_page
from src.figures import get_barchart_sources, get_map_count_marches
from src.utils import lf, meta_content
from src.utils import df, meta_content
name = "Statistiques"
@@ -37,13 +37,13 @@ layout = [
L'ajout de nouvelles plateformes [est en cours](https://github.com/ColinMaudry/decp-processing/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22source%20de%20donn%C3%A9es%22),
toutes les [contributions](/a-propos#contribuer) sont les bienvenues pour atteindre l'exhaustivité.
"""),
dcc.Graph(figure=get_map_count_marches(lf)),
dcc.Graph(figure=get_map_count_marches(df)),
dcc.Graph(
figure=get_barchart_sources(lf, "dateNotification")
figure=get_barchart_sources(df, "dateNotification")
),
dcc.Graph(
figure=get_barchart_sources(
lf, "datePublicationDonnees"
df, "datePublicationDonnees"
)
),
],
src/pages/titulaire.py
+10 -9
View File
@@ -6,11 +6,11 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register
from src.figures import point_on_map
from src.utils import (
add_links_in_dict,
df,
format_montant,
format_number,
get_annuaire_data,
get_departement_region,
lf,
meta_content,
setup_table_columns,
)
@@ -148,22 +148,22 @@ def update_titulaire_infos(url):
Input(component_id="titulaire_data", component_property="data"),
)
def update_titulaire_stats(data):
df = pl.DataFrame(data)
if df.height == 0:
df = pl.DataFrame(schema=lf.collect_schema())
df_marches = df.unique("uid")
dff = pl.DataFrame(data)
if dff.height == 0:
dff = pl.DataFrame(schema=dff.collect_schema())
df_marches = dff.unique("uid")
nb_marches = format_number(df_marches.height)
# somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item()))
marches_remportes = [html.Strong(nb_marches), " marchés et accord-cadres remportés"]
# + ", pour un total de ", html.Strong(somme_marches + " €")]
del df_marches
nb_acheteurs = df.unique("acheteur_id").height
nb_acheteurs = dff.unique("acheteur_id").height
nb_acheteurs = [
html.Strong(format_number(nb_acheteurs)),
" titulaires (SIRET) différents",
]
del df
del dff
return marches_remportes, nb_acheteurs
@@ -173,9 +173,10 @@ def update_titulaire_stats(data):
Input(component_id="url", component_property="pathname"),
Input(component_id="titulaire_year", component_property="value"),
)
def get_titulaire_marches_data(url, titulaire_year: str) -> pl.LazyFrame:
def get_titulaire_marches_data(url, titulaire_year: str) -> list[dict]:
titulaire_siret = url.split("/")[-1]
lff = lf.filter(
lff = df.lazy()
lff = lff.filter(
(pl.col("titulaire_id") == titulaire_siret)
& (pl.col("titulaire_typeIdentifiant") == "SIRET")
)
src/utils.py
+8 -6
View File
@@ -157,7 +157,7 @@ def get_annuaire_data(siret: str) -> dict:
return response.json()["results"][0]
def get_decp_data() -> pl.LazyFrame:
def get_decp_data() -> pl.DataFrame:
# Chargement du fichier parquet
# Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment.
# On utilise polars pour la rapidité et la facilité de manipulation des données.
@@ -186,7 +186,7 @@ def get_decp_data() -> pl.LazyFrame:
# ça génère une erreur dans la page acheteur (acheteur_data.table) :
# AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import)
return lff
return lff.collect()
def get_departements() -> dict:
@@ -206,14 +206,16 @@ def get_departement_region(code_postal):
def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame:
debug = os.getenv("DEVELOPMENT", "False").lower() == "true"
schema = lff.collect_schema()
filtering_expressions = filter_query.split(" && ")
for filter_part in filtering_expressions:
col_name, operator, filter_value = split_filter_part(filter_part)
col_type = str(schema[col_name])
print("filter_value:", filter_value)
print("filter_value_type:", type(filter_value))
print("col_type:", col_type)
if debug:
print("filter_value:", filter_value)
print("filter_value_type:", type(filter_value))
print("col_type:", col_type)
if col_type == "Date":
# Convertir la colonne en chaînes de caractères
@@ -293,7 +295,7 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup
return columns, tooltip
lf = get_decp_data()
df: pl.DataFrame = get_decp_data()
departements = get_departements()
domain_name = (
"test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info"