Merge branch 'feature/share_filtering_datatables' into dev

README.md

@@ -1,8 +1,7 @@
 # decp.info
 
-> v2.2.2
+> v2.3.0
-
+> Outil d'exploration et de téléchargement des données essentielles de la commande publique.
-Outil d'exploration et de téléchargement des données essentielles de la commande publique.
 
 => [decp.info](https://decp.info)
 

pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "decp.info"
 description = "Interface d'exploration et d'analyse des marchés publics français."
-version = "2.2.2"
+version = "2.3.0"
 requires-python = ">= 3.10"
 authors = [
     { name = "Colin Maudry", email = "colin+decp@maudry.com" }
 ]
 dependencies = [
-    "dash==3.2.0",
+    "dash==3.3.0",
     "dash[compress]",
     "polars",
     "gunicorn",

src/assets/style.css

@@ -133,6 +133,12 @@ td[data-dash-column="objet"],td[data-dash-column="titulaire_nom"],td[data-dash-c
 .marches_table {
   font-family: "Open Sans", sans-serif;
 }
+
+.marches_table.stuck {
+  position: relative;
+  right: 200px;
+}
+
 .marches_table .cell-table tr:nth-child(even) td {
   background-color: #feeeee;
   font-family: "Open Sans", sans-serif;
@@ -212,7 +218,7 @@ summary > h3 {
 }
 
 #_pages_content {
-  padding-top: 28px;
+  padding: 28px 24px 0 24px;
 }
 
 /* Vue acheteur/titulaire/recherche    */

src/pages/acheteur.py

@@ -6,14 +6,16 @@ from dash import Input, Output, State, callback, dcc, html, register_page
 from src.callbacks import get_top_org_table
 from src.figures import DataTable, point_on_map
 from src.utils import (
-    add_links_in_dict,
     df,
+    filter_table_data,
     format_number,
-    format_values,
     get_annuaire_data,
+    get_button_properties,
+    get_default_hidden_columns,
     get_departement_region,
     meta_content,
-    setup_table_columns,
+    prepare_table_data,
+    sort_table_data,
 )
 
 register_page(
@@ -26,11 +28,22 @@ register_page(
     order=5,
 )
 
+datatable = html.Div(
+    className="marches_table",
+    children=DataTable(
+        dtid="acheteur_datatable",
+        page_action="custom",
+        filter_action="custom",
+        sort_action="custom",
+        page_size=10,
+        hidden_columns=get_default_hidden_columns(page="acheteur"),
+    ),
+)
+
 layout = [
     dcc.Store(id="acheteur_data", storage_type="memory"),
     dcc.Location(id="url", refresh="callback-nav"),
     html.Div(
-        className="container",
         children=[
             html.Div(
                 className="wrapper",
@@ -84,9 +97,9 @@ layout = [
                             html.P(id="acheteur_titulaires_differents"),
                             html.Button(
                                 "Téléchargement au format Excel",
-                                id="btn-download-acheteur-data",
+                                id="btn-download-data-acheteur",
                             ),
-                            dcc.Download(id="download-acheteur-data"),
+                            dcc.Download(id="download-data-acheteur"),
                         ],
                     ),
                     html.Div(className="org_map", id="acheteur_map"),
@@ -101,7 +114,26 @@ layout = [
             ),
             # récupérer les données de l'acheteur sur l'api annuaire
             html.H3("Derniers marchés publics attribués"),
-            html.Div(id="acheteur_last_marches", children=""),
+            dcc.Loading(
+                overlay_style={"visibility": "visible", "filter": "blur(2px)"},
+                id="loading-home",
+                type="default",
+                children=[
+                    html.Div(
+                        [
+                            html.P("lignes", id="acheteur_nb_rows"),
+                            html.Button(
+                                "Téléchargement désactivé au-delà de 65 000 lignes",
+                                id="btn-download-filtered-data-acheteur",
+                                disabled=True,
+                            ),
+                            dcc.Download(id="acheteur-download-filtered-data"),
+                        ],
+                        className="table-menu",
+                    ),
+                    datatable,
+                ],
+            ),
         ],
     ),
 ]
@@ -176,68 +208,49 @@ def update_acheteur_stats(data):
 
 @callback(
     Output(component_id="acheteur_data", component_property="data"),
+    Output("btn-download-data-acheteur", "disabled"),
+    Output("btn-download-data-acheteur", "children"),
+    Output("btn-download-data-acheteur", "title"),
     Input(component_id="url", component_property="pathname"),
     Input(component_id="acheteur_year", component_property="value"),
 )
-def get_acheteur_marches_data(url, acheteur_year: str) -> list[dict]:
+def get_acheteur_marches_data(url, acheteur_year: str) -> tuple:
     acheteur_siret = url.split("/")[-1]
     lff = df.lazy()
     lff = lff.filter(pl.col("acheteur_id") == acheteur_siret)
-    lff = lff.select(
-        "id",
-        "uid",
-        "objet",
-        "dateNotification",
-        "titulaire_id",
-        "titulaire_typeIdentifiant",
-        "titulaire_nom",
-        "distance",
-        "montant",
-        "codeCPV",
-        "dureeMois",
-    )
     if acheteur_year and acheteur_year != "Toutes":
-        lff = lff.filter(
+        acheteur_year = int(acheteur_year)
-            pl.col("dateNotification").cast(pl.String).str.starts_with(acheteur_year)
+        lff = lff.filter(pl.col("dateNotification").dt.year() == acheteur_year)
-        )
+    lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True)
-    lff = lff.sort(["dateNotification", "id"], descending=True, nulls_last=True)
+    dff: pl.DataFrame = lff.collect(engine="streaming")
+    download_disabled, download_text, download_title = get_button_properties(dff.height)
 
-    data = lff.collect(engine="streaming").to_dicts()
+    data = dff.to_dicts()
-    return data
+    return data, download_disabled, download_text, download_title
 
 
 @callback(
-    Output(component_id="acheteur_last_marches", component_property="children"),
+    Output("acheteur_datatable", "data"),
-    Input(component_id="acheteur_data", component_property="data"),
+    Output("acheteur_datatable", "columns"),
+    Output("acheteur_datatable", "tooltip_header"),
+    Output("acheteur_datatable", "data_timestamp"),
+    Output("acheteur_nb_rows", "children"),
+    Output("btn-download-filtered-data-acheteur", "disabled"),
+    Output("btn-download-filtered-data-acheteur", "children"),
+    Output("btn-download-filtered-data-acheteur", "title"),
+    Input("acheteur_data", "data"),
+    Input("acheteur_datatable", "page_current"),
+    Input("acheteur_datatable", "page_size"),
+    Input("acheteur_datatable", "filter_query"),
+    Input("acheteur_datatable", "sort_by"),
+    State("acheteur_datatable", "data_timestamp"),
 )
-def get_last_marches_table(data) -> html.Div:
+def get_last_marches_data(
-    dff = pl.DataFrame(data)
+    data, page_current, page_size, filter_query, sort_by, data_timestamp
-    if dff.height == 0:
+) -> tuple:
-        return html.Div(html.P("Aucun marché trouvé."))
+    return prepare_table_data(
-    dff = dff.cast(pl.String)
+        data, data_timestamp, filter_query, page_current, page_size, sort_by
-    dff = dff.fill_null("")
-    dff = format_values(dff)
-    columns, tooltip = setup_table_columns(
-        dff,
-        hideable=False,
-        exclude=["titulaire_id", "titulaire_typeIdentifiant", "uid"],
     )
-    data = dff.to_dicts()
-    data = add_links_in_dict(data, "titulaire")
-
-    table = html.Div(
-        className="marches_table",
-        children=DataTable(
-            dtid="acheteur_datatable",
-            data=data,
-            page_action="native",
-            filter_action="native",
-            columns=columns,
-            tooltip_header=tooltip,
-            page_size=10,
-        ),
-    )
-    return table
 
 
 @callback(
@@ -249,8 +262,8 @@ def get_top_titulaires(data):
 
 
 @callback(
-    Output("download-acheteur-data", "data"),
+    Output("download-data-acheteur", "data"),
-    Input("btn-download-acheteur-data", "n_clicks"),
+    Input("btn-download-data-acheteur", "n_clicks"),
     State(component_id="acheteur_data", component_property="data"),
     State(component_id="acheteur_nom", component_property="children"),
     State(component_id="acheteur_year", component_property="value"),
@@ -271,3 +284,39 @@ def download_acheteur_data(
 
     date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
     return dcc.send_bytes(to_bytes, filename=f"decp_{acheteur_nom}_{date}.xlsx")
+
+
+@callback(
+    Output("acheteur-download-filtered-data", "data"),
+    State("acheteur_data", "data"),
+    Input("btn-download-filtered-data-acheteur", "n_clicks"),
+    State("acheteur_nom", "children"),
+    State("acheteur_datatable", "filter_query"),
+    State("acheteur_datatable", "sort_by"),
+    State("acheteur_datatable", "hidden_columns"),
+    prevent_initial_call=True,
+)
+def download_filtered_acheteur_data(
+    data, n_clicks, acheteur_nom, filter_query, sort_by, hidden_columns: list = None
+):
+    lff: pl.LazyFrame = pl.LazyFrame(
+        data
+    )  # start from the full acheteur data, not from paginated table data
+
+    # Les colonnes masquées sont supprimées
+    if hidden_columns:
+        lff = lff.drop(hidden_columns)
+
+    if filter_query:
+        lff = filter_table_data(lff, filter_query)
+
+    if len(sort_by) > 0:
+        lff = sort_table_data(lff, sort_by)
+
+    def to_bytes(buffer):
+        lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
+
+    date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+    return dcc.send_bytes(
+        to_bytes, filename=f"decp_filtrées_{acheteur_nom}_{date}.xlsx"
+    )

src/pages/tableau.py

@@ -6,22 +6,17 @@ from dash import Input, Output, State, callback, dcc, html, register_page
 
 from src.figures import DataTable
 from src.utils import (
-    add_links,
-    add_resource_link,
     df,
     filter_table_data,
-    format_number,
-    format_values,
     get_default_hidden_columns,
     meta_content,
-    setup_table_columns,
     sort_table_data,
 )
+from utils import prepare_table_data
 
 update_date = os.path.getmtime(os.getenv("DATA_FILE_PARQUET_PATH"))
 update_date = datetime.fromtimestamp(update_date).strftime("%d/%m/%Y")
 
-schema = df.collect_schema()
 
 name = "Tableau"
 register_page(
@@ -42,7 +37,7 @@ datatable = html.Div(
         page_action="custom",
         filter_action="custom",
         sort_action="custom",
-        hidden_columns=get_default_hidden_columns(schema),
+        hidden_columns=get_default_hidden_columns(None),
     ),
 )
 
@@ -138,65 +133,8 @@ layout = [
     State("table", "data_timestamp"),
 )
 def update_table(page_current, page_size, filter_query, sort_by, data_timestamp):
-    if os.getenv("DEVELOPMENT").lower() == "true":
+    return prepare_table_data(
-        print(" + + + + + + + + + + + + + + + + + + ")
+        None, data_timestamp, filter_query, page_current, page_size, sort_by
-
-    # Application des filtres
-    lff: pl.LazyFrame = df.lazy()  # start from the original data
-    if filter_query:
-        lff = filter_table_data(lff, filter_query)
-
-    if len(sort_by) > 0:
-        lff = sort_table_data(lff, sort_by)
-
-    # Matérialisation des filtres
-    dff: pl.DataFrame = lff.collect()
-
-    height = dff.height
-    nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)"
-
-    # Pagination des données
-    start_row = page_current * page_size
-    # end_row = (page_current + 1) * page_size
-    dff = dff.slice(start_row, page_size)
-
-    # Tout devient string
-    dff = dff.cast(pl.String)
-
-    # Remplace les strings null par "", mais pas les numeric null
-    dff = dff.fill_null("")
-
-    # Ajout des liens vers l'annuaire des entreprises
-    dff = add_links(dff)
-
-    # Ajout des liens vers les fichiers Open Data
-    dff = add_resource_link(dff)
-
-    # Formatage des montants
-    dff = format_values(dff)
-
-    columns, tooltip = setup_table_columns(dff)
-
-    dicts = dff.to_dicts()
-
-    if height > 65000:
-        download_disabled = True
-        download_text = "Téléchargement désactivé au-delà de 65 000 lignes"
-        download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution."
-    else:
-        download_disabled = False
-        download_text = "Télécharger au format Excel"
-        download_title = ""
-
-    return (
-        dicts,
-        columns,
-        tooltip,
-        data_timestamp + 1,
-        nb_rows,
-        download_disabled,
-        download_text,
-        download_title,
     )
 
 

src/pages/titulaire.py

@@ -6,14 +6,16 @@ from dash import Input, Output, State, callback, dcc, html, register_page
 from src.callbacks import get_top_org_table
 from src.figures import DataTable, point_on_map
 from src.utils import (
-    add_links_in_dict,
     df,
+    filter_table_data,
     format_number,
-    format_values,
     get_annuaire_data,
+    get_button_properties,
+    get_default_hidden_columns,
     get_departement_region,
     meta_content,
-    setup_table_columns,
+    prepare_table_data,
+    sort_table_data,
 )
 
 register_page(
@@ -26,13 +28,22 @@ register_page(
     order=5,
 )
 
-# 21690123100011
+datatable = html.Div(
+    className="marches_table",
+    children=DataTable(
+        dtid="titulaire_datatable",
+        page_action="custom",
+        filter_action="custom",
+        sort_action="custom",
+        page_size=10,
+        hidden_columns=get_default_hidden_columns(page="titulaire"),
+    ),
+)
 
 layout = [
     dcc.Store(id="titulaire_data", storage_type="memory"),
     dcc.Location(id="url", refresh="callback-nav"),
     html.Div(
-        className="container",
         children=[
             html.Div(
                 className="wrapper",
@@ -86,9 +97,9 @@ layout = [
                             html.P(id="titulaire_acheteurs_differents"),
                             html.Button(
                                 "Téléchargement au format Excel",
-                                id="btn-download-titulaire-data",
+                                id="btn-download-data-titulaire",
                             ),
-                            dcc.Download(id="download-titulaire-data"),
+                            dcc.Download(id="download-data-titulaire"),
                         ],
                     ),
                     html.Div(className="org_map", id="titulaire_map"),
@@ -103,7 +114,26 @@ layout = [
             ),
             # récupérer les données de l'acheteur sur l'api annuaire
             html.H3("Derniers marchés publics remportés"),
-            html.Div(id="titulaire_last_marches", children=""),
+            dcc.Loading(
+                overlay_style={"visibility": "visible", "filter": "blur(2px)"},
+                id="loading-home",
+                type="default",
+                children=[
+                    html.Div(
+                        [
+                            html.P("lignes", id="titulaire_nb_rows"),
+                            html.Button(
+                                "Téléchargement désactivé au-delà de 65 000 lignes",
+                                id="btn-download-filtered-data-titulaire",
+                                disabled=True,
+                            ),
+                            dcc.Download(id="titulaire-download-filtered-data"),
+                        ],
+                        className="table-menu",
+                    ),
+                    datatable,
+                ],
+            ),
         ],
     ),
 ]
@@ -158,88 +188,78 @@ def update_titulaire_infos(url):
 def update_titulaire_stats(data):
     dff = pl.DataFrame(data)
     if dff.height == 0:
-        dff = pl.DataFrame(schema=dff.collect_schema())
+        nb_marches = 0
-    df_marches = dff.unique("uid")
+        nb_acheteurs = 0
-    nb_marches = format_number(df_marches.height)
+    else:
-    # somme_marches = format_number(int(df_marches.select(pl.sum("montant")).item()))
+        df_marches = dff.unique("uid")
-    marches_remportes = [html.Strong(nb_marches), " marchés et accord-cadres remportés"]
+        nb_marches = format_number(df_marches.height)
-    # + ", pour un total de ", html.Strong(somme_marches + " €")]
+        nb_acheteurs = dff.unique("acheteur_id").height
-    del df_marches
 
-    nb_acheteurs = dff.unique("acheteur_id").height
+    texte_marches_remportes = [
-    nb_acheteurs = [
+        html.Strong(nb_marches),
+        " marchés et accord-cadres remportés",
+    ]
+    # + ", pour un total de ", html.Strong(somme_marches + " €")]
+
+    texte_nb_acheteurs = [
         html.Strong(format_number(nb_acheteurs)),
         " acheteurs (SIRET) différents",
     ]
-    del dff
 
-    return marches_remportes, nb_acheteurs
+    return texte_marches_remportes, texte_nb_acheteurs
 
 
 @callback(
     Output(component_id="titulaire_data", component_property="data"),
+    Output("btn-download-data-titulaire", "disabled"),
+    Output("btn-download-data-titulaire", "children"),
+    Output("btn-download-data-titulaire", "title"),
     Input(component_id="url", component_property="pathname"),
     Input(component_id="titulaire_year", component_property="value"),
 )
-def get_titulaire_marches_data(url, titulaire_year: str) -> list[dict]:
+def get_titulaire_marches_data(url, titulaire_year: str) -> tuple:
     titulaire_siret = url.split("/")[-1]
     lff = df.lazy()
     lff = lff.filter(
         (pl.col("titulaire_id") == titulaire_siret)
         & (pl.col("titulaire_typeIdentifiant") == "SIRET")
     )
-    lff = lff.select(
-        "id",
-        "uid",
-        "objet",
-        "dateNotification",
-        "acheteur_id",
-        "acheteur_nom",
-        "distance",
-        "montant",
-        "codeCPV",
-        "dureeMois",
-    )
     if titulaire_year and titulaire_year != "Toutes":
         lff = lff.filter(
             pl.col("dateNotification").cast(pl.String).str.starts_with(titulaire_year)
         )
     lff = lff.sort(["dateNotification", "uid"], descending=True, nulls_last=True)
+    lff = lff.fill_null("")
 
-    data = lff.collect(engine="streaming").to_dicts()
+    dff: pl.DataFrame = lff.collect(engine="streaming")
-    return data
+    download_disabled, download_text, download_title = get_button_properties(dff.height)
+
+    data = dff.to_dicts()
+    return data, download_disabled, download_text, download_title
 
 
 @callback(
-    Output(component_id="titulaire_last_marches", component_property="children"),
+    Output("titulaire_datatable", "data"),
-    Input(component_id="titulaire_data", component_property="data"),
+    Output("titulaire_datatable", "columns"),
+    Output("titulaire_datatable", "tooltip_header"),
+    Output("titulaire_datatable", "data_timestamp"),
+    Output("titulaire_nb_rows", "children"),
+    Output("btn-download-filtered-data-titulaire", "disabled"),
+    Output("btn-download-filtered-data-titulaire", "children"),
+    Output("btn-download-filtered-data-titulaire", "title"),
+    Input("titulaire_data", "data"),
+    Input("titulaire_datatable", "page_current"),
+    Input("titulaire_datatable", "page_size"),
+    Input("titulaire_datatable", "filter_query"),
+    Input("titulaire_datatable", "sort_by"),
+    State("titulaire_datatable", "data_timestamp"),
 )
-def get_last_marches_table(data) -> html.Div:
+def get_last_marches_data(
-    dff = pl.DataFrame(data)
+    data, page_current, page_size, filter_query, sort_by, data_timestamp
-    dff = dff.cast(pl.String)
+) -> list[dict]:
-    dff = dff.fill_null("")
+    return prepare_table_data(
-    dff = format_values(dff)
+        data, data_timestamp, filter_query, page_current, page_size, sort_by
-    columns, tooltip = setup_table_columns(
-        dff, hideable=False, exclude=["acheteur_id", "id"]
     )
-    data = dff.to_dicts()
-    # Idéalement on utiliserait add_org_links(), mais le résultat attendu
-    # est différent de home.py (Tableau)
-    data = add_links_in_dict(data, "acheteur")
-
-    table = html.Div(
-        className="marches_table",
-        children=DataTable(
-            dtid="titulaire_data_table",
-            data=data,
-            page_action="native",
-            filter_action="native",
-            columns=columns,
-            tooltip_header=tooltip,
-            page_size=10,
-        ),
-    )
-    return table
 
 
 @callback(
@@ -251,8 +271,8 @@ def get_top_acheteurs(data):
 
 
 @callback(
-    Output("download-titulaire-data", "data"),
+    Output("download-data-titulaire", "data"),
-    Input("btn-download-titulaire-data", "n_clicks"),
+    Input("btn-download-data-titulaire", "n_clicks"),
     State(component_id="titulaire_data", component_property="data"),
     State(component_id="titulaire_nom", component_property="children"),
     State(component_id="titulaire_year", component_property="value"),
@@ -273,3 +293,39 @@ def download_titulaire_data(
 
     date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
     return dcc.send_bytes(to_bytes, filename=f"decp_{titulaire_nom}_{date}.xlsx")
+
+
+@callback(
+    Output("titulaire-download-filtered-data", "data"),
+    State("titulaire_data", "data"),
+    Input("btn-download-filtered-data-titulaire", "n_clicks"),
+    State("titulaire_nom", "children"),
+    State("titulaire_datatable", "filter_query"),
+    State("titulaire_datatable", "sort_by"),
+    State("titulaire_datatable", "hidden_columns"),
+    prevent_initial_call=True,
+)
+def download_filtered_titulaire_data(
+    data, n_clicks, titulaire_nom, filter_query, sort_by, hidden_columns: list = None
+):
+    lff: pl.LazyFrame = pl.LazyFrame(
+        data
+    )  # start from the full titulaire data, not from paginated table data
+
+    # Les colonnes masquées sont supprimées
+    if hidden_columns:
+        lff = lff.drop(hidden_columns)
+
+    if filter_query:
+        lff = filter_table_data(lff, filter_query)
+
+    if len(sort_by) > 0:
+        lff = sort_table_data(lff, sort_by)
+
+    def to_bytes(buffer):
+        lff.collect(engine="streaming").write_excel(buffer, worksheet="DECP")
+
+    date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+    return dcc.send_bytes(
+        to_bytes, filename=f"decp_filtrées_{titulaire_nom}_{date}.xlsx"
+    )

src/utils.py

@@ -7,7 +7,6 @@ from time import localtime, sleep
 import polars as pl
 import polars.selectors as cs
 from httpx import get, post
-from polars import Schema
 from polars.exceptions import ComputeError
 from unidecode import unidecode
 
@@ -331,7 +330,7 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup
             continue
         column_object = data_schema.get(column_id)
         if column_object:
-            column_name = column_object.get("title", column_id)
+            column_name = column_object.get("title")
         else:
             column_name = column_id
 
@@ -356,18 +355,47 @@ def setup_table_columns(dff, hideable: bool = True, exclude: list = None) -> tup
     return columns, tooltip
 
 
-def get_default_hidden_columns(schema: Schema):
+def get_default_hidden_columns(page):
-    displayed_columns = os.getenv("DISPLAYED_COLUMNS")
+    if page == "acheteur":
+        displayed_columns = [
+            "uid",
+            "objet",
+            "dateNotification",
+            "titulaire_id",
+            "titulaire_typeIdentifiant",
+            "titulaire_nom",
+            "distance",
+            "montant",
+            "codeCPV",
+            "dureeRestanteMois",
+        ]
+    elif page == "titulaire":
+        displayed_columns = [
+            "uid",
+            "objet",
+            "dateNotification",
+            "acheteur_id",
+            "acheteur_nom",
+            "distance",
+            "montant",
+            "codeCPV",
+            "dureeRestanteMois",
+        ]
+    else:
+        displayed_columns = os.getenv("DISPLAYED_COLUMNS")
+        if displayed_columns is None:
+            raise ValueError("DISPLAYED_COLUMNS n'est pas configuré")
+        else:
+            displayed_columns = displayed_columns.replace(" ", "").split(",")
+
     hidden_columns = []
-    if displayed_columns:
+
-        displayed_columns = displayed_columns.replace(" ", "").split(",")
+    for col in schema.names():
-        for col in schema.names():
+        if col in displayed_columns:
-            if col in displayed_columns:
+            continue
-                continue
+        else:
-            else:
+            hidden_columns.append(col)
-                hidden_columns.append(col)
+    return hidden_columns
-        return hidden_columns
-    raise ValueError("DISPLAYED_COLUMNS n'est pas configuré")
 
 
 def get_data_schema() -> dict:
@@ -503,9 +531,111 @@ def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
     return dff
 
 
+def prepare_table_data(
+    data, data_timestamp, filter_query, page_current, page_size, sort_by
+):
+    """
+    Fonction de préparation des données pour les datatables, afin de permettre une gestion fine des logiques,
+    notamment pour les filtres et les tris.
+    :param data
+    :param data_timestamp:
+    :param filter_query:
+    :param page_current:
+    :param page_size:
+    :param sort_by:
+    :return:
+    """
+
+    if os.getenv("DEVELOPMENT").lower() == "true":
+        print(" + + + + + + + + + + + + + + + + + + ")
+
+    # Récupération des données
+    if isinstance(data, list):
+        lff: pl.LazyFrame = pl.LazyFrame(data)
+    else:
+        lff: pl.LazyFrame = df.lazy()  # start from the original data
+
+    # Application des filtres
+    if filter_query:
+        lff = filter_table_data(lff, filter_query)
+
+    # Application des tris
+    if len(sort_by) > 0:
+        lff = sort_table_data(lff, sort_by)
+
+    # Matérialisation des filtres
+    dff: pl.DataFrame = lff.collect()
+    height = dff.height
+
+    if height > 0:
+        nb_rows = f"{format_number(height)} lignes ({format_number(dff.select('uid').unique().height)} marchés)"
+    else:
+        nb_rows = "0 lignes (0 marchés)"
+
+    # Pagination des données
+    start_row = page_current * page_size
+    # end_row = (page_current + 1) * page_size
+    dff = dff.slice(start_row, page_size)
+
+    # Tout devient string
+    dff = dff.cast(pl.String)
+
+    # Remplace les strings null par "", mais pas les numeric null
+    dff = dff.fill_null("")
+
+    # Ajout des liens vers l'annuaire des entreprises
+    dff = add_links(dff)
+
+    # Ajout des liens vers les fichiers Open Data
+    if "sourceFile" in dff.columns:
+        dff = add_resource_link(dff)
+
+    # Formatage des montants
+    if height > 0:
+        dff = format_values(dff)
+
+    # Récupération des colonnes et tooltip
+    columns, tooltip = setup_table_columns(dff)
+
+    dicts = dff.to_dicts()
+
+    # Propriétés du bouton de téléchargement
+    download_disabled, download_text, download_title = get_button_properties(height)
+
+    return (
+        dicts,
+        columns,
+        tooltip,
+        data_timestamp + 1,
+        nb_rows,
+        download_disabled,
+        download_text,
+        download_title,
+    )
+
+
+def get_button_properties(height):
+    if height > 65000:
+        download_disabled = True
+        download_text = "Téléchargement désactivé au-delà de 65 000 lignes"
+        download_title = "Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul. Contactez-moi pour me présenter votre besoin en téléchargement afin que je puisse adapter la solution."
+    elif height == 0:
+        download_disabled = True
+        download_text = "Pas de données à télécharger"
+        download_title = ""
+    else:
+        download_disabled = False
+        download_text = "Télécharger au format Excel"
+        download_title = ""
+    return download_disabled, download_text, download_title
+
+
 df: pl.DataFrame = get_decp_data()
+schema = df.collect_schema()
+
 df_acheteurs = get_org_data(df, "acheteur")
 df_titulaires = get_org_data(df, "titulaire")
+
 departements = get_departements()
 domain_name = (
     "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info"