diff --git a/src/figures.py b/src/figures.py index 25e1b9e..ec5eda1 100644 --- a/src/figures.py +++ b/src/figures.py @@ -5,7 +5,8 @@ import polars as pl from dash import dash_table, dcc, html -def get_map_count_marches(lf: pl.LazyFrame): +def get_map_count_marches(df: pl.DataFrame): + lf = df.lazy() lf = lf.with_columns( pl.col("lieuExecution_code").str.head(2).str.zfill(2).alias("Département") ) @@ -26,7 +27,7 @@ def get_map_count_marches(lf: pl.LazyFrame): for f in departements["features"]: f["id"] = f["properties"]["code"] - df = lf.collect() + df = lf.collect(engine="streaming") fig = px.choropleth( df, @@ -54,7 +55,8 @@ def get_map_count_marches(lf: pl.LazyFrame): return fig -def get_barchart_sources(lf: pl.LazyFrame, type_date: str): +def get_barchart_sources(df: pl.DataFrame, type_date: str): + lf = df.lazy() labels = { "dateNotification": "notification", "datePublicationDonnees": "publication des données", @@ -97,7 +99,7 @@ def get_barchart_sources(lf: pl.LazyFrame, type_date: str): # ) lf = lf.sort(by=["sourceDataset"], descending=False) - df: pl.DataFrame = lf.collect() + df: pl.DataFrame = lf.collect(engine="streaming") fig = px.bar( df, diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 4245b00..3da7e7a 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -6,11 +6,11 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.figures import point_on_map from src.utils import ( add_links_in_dict, + df, format_montant, format_number, get_annuaire_data, get_departement_region, - lf, meta_content, setup_table_columns, ) @@ -148,7 +148,7 @@ def update_acheteur_infos(url): def update_acheteur_stats(data): df = pl.DataFrame(data) if df.height == 0: - df = pl.DataFrame(schema=lf.collect_schema()) + df = pl.DataFrame(schema=df.collect_schema()) df_marches = df.unique("id") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) @@ -171,9 +171,10 @@ def update_acheteur_stats(data): Input(component_id="url", component_property="pathname"), Input(component_id="acheteur_year", component_property="value"), ) -def get_acheteur_marches_data(url, acheteur_year: str) -> pl.LazyFrame: +def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]: acheteur_siret = url.split("/")[-1] - lff = lf.filter(pl.col("acheteur_id") == acheteur_siret) + lff = df.lazy() + lff = lff.filter(pl.col("acheteur_id") == acheteur_siret) lff = lff.fill_null("") lff = lff.select( "id", diff --git a/src/pages/home.py b/src/pages/home.py index 164f12a..cdef299 100644 --- a/src/pages/home.py +++ b/src/pages/home.py @@ -7,10 +7,10 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.utils import ( add_links, add_resource_link, + df, filter_table_data, format_montant, format_number, - lf, meta_content, setup_table_columns, sort_table_data, @@ -19,7 +19,7 @@ from src.utils import ( update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH")) update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y") -schema = lf.collect_schema() +schema = df.collect_schema() name = "Tableau" register_page( @@ -171,10 +171,11 @@ layout = [ State("table", "data_timestamp"), ) def update_table(page_current, page_size, filter_query, sort_by, data_timestamp): - print(" + + + + + + + + + + + + + + + + + + ") + if os.getenv("DEVELOPMENT").lower() == "true": + print(" + + + + + + + + + + + + + + + + + + ") # Application des filtres - lff: pl.LazyFrame = lf # start from the original data + lff: pl.LazyFrame = df.lazy() # start from the original data if filter_query: lff = filter_table_data(lff, filter_query) @@ -238,7 +239,7 @@ def update_table(page_current, page_size, filter_query, sort_by, data_timestamp) prevent_initial_call=True, ) def download_data(n_clicks, filter_query, sort_by, hidden_columns: list = None): - lff: pl.LazyFrame = lf # start from the original data + lff: pl.LazyFrame = df # start from the original data # Les colonnes masquées sont supprimées if hidden_columns: diff --git a/src/pages/marche.py b/src/pages/marche.py index 88c1baf..798d033 100644 --- a/src/pages/marche.py +++ b/src/pages/marche.py @@ -4,7 +4,7 @@ import polars as pl from dash import Input, Output, callback, dcc, html, register_page from polars import selectors as cs -from src.utils import data_schema, format_montant, lf, meta_content +from src.utils import data_schema, df, format_montant, meta_content register_page( __name__, @@ -62,11 +62,12 @@ layout = [ Output("titulaires_data", "data"), Input(component_id="url", component_property="pathname"), ) -def get_marche_data(url): +def get_marche_data(url) -> tuple[dict, list]: marche_uid = url.split("/")[-1] - # Récupération des données du marché à partir du lf global - lff = lf.filter(pl.col("uid") == pl.lit(marche_uid)) + # Récupération des données du marché à partir du df global + lff = df.lazy() + lff = lff.filter(pl.col("uid") == pl.lit(marche_uid)) # Données des titulaires du marché dff_titulaires = lff.select(cs.starts_with("titulaire")).collect(engine="streaming") diff --git a/src/pages/statistiques.py b/src/pages/statistiques.py index a55b8c8..1caa4d4 100644 --- a/src/pages/statistiques.py +++ b/src/pages/statistiques.py @@ -1,7 +1,7 @@ from dash import dcc, html, register_page from src.figures import get_barchart_sources, get_map_count_marches -from src.utils import lf, meta_content +from src.utils import df, meta_content name = "Statistiques" @@ -37,13 +37,13 @@ layout = [ L'ajout de nouvelles plateformes [est en cours](https://github.com/ColinMaudry/decp-processing/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22source%20de%20donn%C3%A9es%22), toutes les [contributions](/a-propos#contribuer) sont les bienvenues pour atteindre l'exhaustivité. """), - dcc.Graph(figure=get_map_count_marches(lf)), + dcc.Graph(figure=get_map_count_marches(df)), dcc.Graph( - figure=get_barchart_sources(lf, "dateNotification") + figure=get_barchart_sources(df, "dateNotification") ), dcc.Graph( figure=get_barchart_sources( - lf, "datePublicationDonnees" + df, "datePublicationDonnees" ) ), ], diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index bca19c4..bd38feb 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -6,11 +6,11 @@ from dash import Input, Output, State, callback, dash_table, dcc, html, register from src.figures import point_on_map from src.utils import ( add_links_in_dict, + df, format_montant, format_number, get_annuaire_data, get_departement_region, - lf, meta_content, setup_table_columns, ) @@ -148,22 +148,22 @@ def update_titulaire_infos(url): Input(component_id="titulaire_data", component_property="data"), ) def update_titulaire_stats(data): - df = pl.DataFrame(data) - if df.height == 0: - df = pl.DataFrame(schema=lf.collect_schema()) - df_marches = df.unique("uid") + dff = pl.DataFrame(data) + if dff.height == 0: + dff = pl.DataFrame(schema=dff.collect_schema()) + df_marches = dff.unique("uid") nb_marches = format_number(df_marches.height) # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item())) marches_remportes = [html.Strong(nb_marches), " marchés et accord-cadres remportés"] # + ", pour un total de ", html.Strong(somme_marches + " €")] del df_marches - nb_acheteurs = df.unique("acheteur_id").height + nb_acheteurs = dff.unique("acheteur_id").height nb_acheteurs = [ html.Strong(format_number(nb_acheteurs)), " titulaires (SIRET) différents", ] - del df + del dff return marches_remportes, nb_acheteurs @@ -173,9 +173,10 @@ def update_titulaire_stats(data): Input(component_id="url", component_property="pathname"), Input(component_id="titulaire_year", component_property="value"), ) -def get_titulaire_marches_data(url, titulaire_year: str) -> pl.LazyFrame: +def get_titulaire_marches_data(url, titulaire_year: str) -> list[dict]: titulaire_siret = url.split("/")[-1] - lff = lf.filter( + lff = df.lazy() + lff = lff.filter( (pl.col("titulaire_id") == titulaire_siret) & (pl.col("titulaire_typeIdentifiant") == "SIRET") ) diff --git a/src/utils.py b/src/utils.py index 4aaa579..466ad9f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -157,7 +157,7 @@ def get_annuaire_data(siret: str) -> dict: return response.json()["results"][0] -def get_decp_data() -> pl.LazyFrame: +def get_decp_data() -> pl.DataFrame: # Chargement du fichier parquet # Le fichier est chargé en mémoire, ce qui est plus rapide qu'une base de données pour le moment. # On utilise polars pour la rapidité et la facilité de manipulation des données. @@ -186,7 +186,7 @@ def get_decp_data() -> pl.LazyFrame: # ça génère une erreur dans la page acheteur (acheteur_data.table) : # AttributeError: partially initialized module 'pandas' has no attribute 'NaT' (most likely due to a circular import) - return lff + return lff.collect() def get_departements() -> dict: @@ -206,14 +206,16 @@ def get_departement_region(code_postal): def filter_table_data(lff: pl.LazyFrame, filter_query: str) -> pl.LazyFrame: + debug = os.getenv("DEVELOPMENT", "False").lower() == "true" schema = lff.collect_schema() filtering_expressions = filter_query.split(" && ") for filter_part in filtering_expressions: col_name, operator, filter_value = split_filter_part(filter_part) col_type = str(schema[col_name]) - print("filter_value:", filter_value) - print("filter_value_type:", type(filter_value)) - print("col_type:", col_type) + if debug: + print("filter_value:", filter_value) + print("filter_value_type:", type(filter_value)) + print("col_type:", col_type) if col_type == "Date": # Convertir la colonne en chaînes de caractères @@ -293,7 +295,7 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup return columns, tooltip -lf = get_decp_data() +df: pl.DataFrame = get_decp_data() departements = get_departements() domain_name = ( "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info"