diff --git a/src/figures.py b/src/figures.py index 2ee558c..faeb638 100644 --- a/src/figures.py +++ b/src/figures.py @@ -13,13 +13,8 @@ from dash import dash_table, dcc, html from dash_extensions.javascript import Namespace from src.db import schema -from src.old_utils import ( - DATA_SCHEMA, - DEPARTEMENTS_GEOJSON, - add_links, - format_number, - setup_table_columns, -) +from src.utils.data import DATA_SCHEMA, DEPARTEMENTS_GEOJSON +from src.utils.table import add_links, format_number, setup_table_columns def get_yearly_statistics(statistics, today_str) -> html.Div: diff --git a/src/pages/a-propos.py b/src/pages/a-propos.py index 9f644d3..91bb01b 100644 --- a/src/pages/a-propos.py +++ b/src/pages/a-propos.py @@ -3,7 +3,7 @@ import os from dash import dcc, html, register_page from src.figures import get_sources_tables -from src.old_utils import META_CONTENT +from src.utils.seo import META_CONTENT NAME = "À propos" diff --git a/src/pages/acheteur.py b/src/pages/acheteur.py index 1913bf9..58e61db 100644 --- a/src/pages/acheteur.py +++ b/src/pages/acheteur.py @@ -24,16 +24,14 @@ from src.figures import ( make_column_picker, point_on_map, ) -from src.old_utils import ( +from src.utils.data import DF_ACHETEURS, get_annuaire_data, get_departement_region +from src.utils.frontend import get_button_properties +from src.utils.seo import META_CONTENT +from src.utils.table import ( COLUMNS, - DF_ACHETEURS, - META_CONTENT, filter_table_data, format_number, - get_annuaire_data, - get_button_properties, get_default_hidden_columns, - get_departement_region, prepare_table_data, sort_table_data, ) diff --git a/src/pages/arbre/departement.py b/src/pages/arbre/departement.py index d6d0358..18c22b0 100644 --- a/src/pages/arbre/departement.py +++ b/src/pages/arbre/departement.py @@ -1,7 +1,7 @@ from dash import Input, Output, callback, dcc, html, register_page from src.db import get_cursor -from src.old_utils import DEPARTEMENTS +from src.utils.data import DEPARTEMENTS NAME = "Département" diff --git a/src/pages/arbre/departements.py b/src/pages/arbre/departements.py index 1ad4c43..c27cdba 100644 --- a/src/pages/arbre/departements.py +++ b/src/pages/arbre/departements.py @@ -1,6 +1,6 @@ from dash import dcc, html, register_page -from src.old_utils import DEPARTEMENTS +from src.utils.data import DEPARTEMENTS NAME = "Départements" diff --git a/src/pages/arbre/liste_marches_org.py b/src/pages/arbre/liste_marches_org.py index 2b1f33d..26cba1c 100644 --- a/src/pages/arbre/liste_marches_org.py +++ b/src/pages/arbre/liste_marches_org.py @@ -2,7 +2,7 @@ import polars as pl from dash import Input, Output, callback, dcc, html, register_page from src.db import get_cursor -from src.old_utils import DF_ACHETEURS, DF_TITULAIRES +from src.utils.data import DF_ACHETEURS, DF_TITULAIRES NAME = "Liste des marchés publics" diff --git a/src/pages/marche.py b/src/pages/marche.py index f191772..3709377 100644 --- a/src/pages/marche.py +++ b/src/pages/marche.py @@ -6,13 +6,9 @@ from dash import Input, Output, callback, dcc, html, register_page from polars import selectors as cs from src.db import query_marches -from src.old_utils import ( - DATA_SCHEMA, - META_CONTENT, - format_values, - make_org_jsonld, - unformat_montant, -) +from src.utils.data import DATA_SCHEMA +from src.utils.seo import META_CONTENT, make_org_jsonld +from src.utils.table import format_values, unformat_montant def get_title(uid: str = None) -> str: diff --git a/src/pages/observatoire.py b/src/pages/observatoire.py index 694658f..a490146 100644 --- a/src/pages/observatoire.py +++ b/src/pages/observatoire.py @@ -30,18 +30,16 @@ from src.figures import ( make_column_picker, make_donut, ) -from src.old_utils import ( - COLUMNS, +from src.utils import logger +from src.utils.data import ( DEPARTEMENTS, DF_ACHETEURS, DF_TITULAIRES, - META_CONTENT, - get_default_hidden_columns, - get_enum_values_as_dict, - logger, prepare_dashboard_data, - prepare_table_data, ) +from src.utils.frontend import get_enum_values_as_dict +from src.utils.seo import META_CONTENT +from src.utils.table import COLUMNS, get_default_hidden_columns, prepare_table_data NAME = "Observatoire" diff --git a/src/pages/recherche.py b/src/pages/recherche.py index 1d9bff4..91a2f3c 100644 --- a/src/pages/recherche.py +++ b/src/pages/recherche.py @@ -2,13 +2,10 @@ import dash_bootstrap_components as dbc from dash import Input, Output, State, callback, dcc, html, register_page from src.figures import DataTable -from src.old_utils import ( - DF_ACHETEURS, - DF_TITULAIRES, - META_CONTENT, - search_org, - setup_table_columns, -) +from src.utils.data import DF_ACHETEURS, DF_TITULAIRES +from src.utils.search import search_org +from src.utils.seo import META_CONTENT +from src.utils.table import setup_table_columns NAME = "Recherche" diff --git a/src/pages/tableau.py b/src/pages/tableau.py index e68e543..723db62 100644 --- a/src/pages/tableau.py +++ b/src/pages/tableau.py @@ -21,13 +21,13 @@ from dash import ( from src.db import query_marches, schema from src.figures import DataTable, make_column_picker -from src.old_utils import ( +from src.utils import logger +from src.utils.seo import META_CONTENT +from src.utils.table import ( COLUMNS, - META_CONTENT, filter_table_data, get_default_hidden_columns, invert_columns, - logger, prepare_table_data, sort_table_data, ) diff --git a/src/pages/titulaire.py b/src/pages/titulaire.py index cecb91f..3347c19 100644 --- a/src/pages/titulaire.py +++ b/src/pages/titulaire.py @@ -23,16 +23,14 @@ from src.figures import ( make_column_picker, point_on_map, ) -from src.old_utils import ( +from src.utils.data import DF_TITULAIRES, get_annuaire_data, get_departement_region +from src.utils.frontend import get_button_properties +from src.utils.seo import META_CONTENT +from src.utils.table import ( COLUMNS, - DF_TITULAIRES, - META_CONTENT, filter_table_data, format_number, - get_annuaire_data, - get_button_properties, get_default_hidden_columns, - get_departement_region, prepare_table_data, sort_table_data, ) diff --git a/src/utils/data.py b/src/utils/data.py new file mode 100644 index 0000000..e4817be --- /dev/null +++ b/src/utils/data.py @@ -0,0 +1,214 @@ +import json +import logging +import os +from collections import OrderedDict +from datetime import datetime, timedelta + +import polars as pl +from httpx import HTTPError, get + +from src.db import get_cursor, schema +from src.utils import logger + +logging.getLogger("httpx").setLevel("WARNING") + + +def get_annuaire_data(siret: str) -> dict: + url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" + try: + response = get(url).raise_for_status() + response = response.json()["results"][0] + except (HTTPError, IndexError): + response = None + logger.warning("Could not fetch data from recherche-entreprises.api.") + return response + + +def get_statistics() -> dict: + return ( + get( + "https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df", + follow_redirects=True, + ) + .raise_for_status() + .json() + ) + + +def get_departements() -> dict: + with open("data/departements.json", "rb") as f: + data = json.load(f) + return data + + +def get_departements_geojson() -> dict: + with open("./data/departements-1000m.geojson") as f: + geojson = json.load(f) + + # Ajout de feature.id + for f in geojson["features"]: + f["id"] = f["properties"]["code"] + + return geojson + + +def get_departement_region(code_postal): + if code_postal > "97000": + code_departement = code_postal[:3] + else: + code_departement = code_postal[:2] + nom_departement = DEPARTEMENTS[code_departement]["departement"] + nom_region = DEPARTEMENTS[code_departement]["region"] + return code_departement, nom_departement, nom_region + + +def get_data_schema() -> dict: + # Récupération du schéma des données tabulaires + path = os.getenv("DATA_SCHEMA_PATH") + if path.startswith("http"): + original_schema: dict = get( + os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True + ).json() + elif os.path.exists(path): + with open(path) as f: + original_schema: dict = json.load(f) + else: + raise Exception(f"Chemin vers le schéma invalide: {path}") + + new_schema = OrderedDict() + + for col in original_schema["fields"]: + new_schema[col["name"]] = col + + return new_schema + + +def prepare_dashboard_data( + lff: pl.LazyFrame, + dashboard_year=None, + dashboard_acheteur_id=None, + dashboard_acheteur_categorie=None, + dashboard_acheteur_departement_code=None, + dashboard_titulaire_id=None, + dashboard_titulaire_categorie=None, + dashboard_titulaire_departement_code=None, + dashboard_marche_type=None, + dashboard_marche_objet=None, + dashboard_marche_code_cpv=None, + dashboard_marche_considerations_sociales=None, + dashboard_marche_considerations_environnementales=None, + dashboard_marche_techniques=None, + dashboard_marche_innovant=None, + dashboard_marche_sous_traitance_declaree=None, + dashboard_montant_min=None, + dashboard_montant_max=None, +) -> pl.LazyFrame: + if dashboard_year: + lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) + else: + lff = lff.filter( + pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) + ) + + if dashboard_acheteur_id: + lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) + else: + if dashboard_acheteur_categorie: + lff = lff.filter( + pl.col("acheteur_categorie") == dashboard_acheteur_categorie + ) + if dashboard_acheteur_departement_code: + lff = lff.filter( + pl.col("acheteur_departement_code").is_in( + dashboard_acheteur_departement_code + ) + ) + + if dashboard_titulaire_id: + lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) + else: + if dashboard_titulaire_categorie: + lff = lff.filter( + pl.col("titulaire_categorie") == dashboard_titulaire_categorie + ) + if dashboard_titulaire_departement_code: + lff = lff.filter( + pl.col("titulaire_departement_code").is_in( + dashboard_titulaire_departement_code + ) + ) + + if dashboard_marche_type: + lff = lff.filter(pl.col("type") == dashboard_marche_type) + + if dashboard_marche_objet: + lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}")) + + if dashboard_marche_code_cpv: + lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv)) + + if dashboard_marche_innovant and dashboard_marche_innovant != "all": + lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant) + + if ( + dashboard_marche_sous_traitance_declaree + and dashboard_marche_sous_traitance_declaree != "all" + ): + lff = lff.filter( + pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree + ) + + if dashboard_marche_techniques: + lff = lff.filter( + pl.col("techniques") + .str.split(", ") + .list.set_intersection(dashboard_marche_techniques) + .list.len() + > 0 + ) + + if dashboard_marche_considerations_sociales: + lff = lff.filter( + pl.col("considerationsSociales") + .str.split(", ") + .list.set_intersection(dashboard_marche_considerations_sociales) + .list.len() + > 0 + ) + + if dashboard_marche_considerations_environnementales: + lff = lff.filter( + pl.col("considerationsEnvironnementales") + .str.split(", ") + .list.set_intersection(dashboard_marche_considerations_environnementales) + .list.len() + > 0 + ) + + if dashboard_montant_min is not None: + lff = lff.filter(pl.col("montant") >= dashboard_montant_min) + + if dashboard_montant_max is not None: + lff = lff.filter(pl.col("montant") <= dashboard_montant_max) + + return lff + + +def build_org_frame(org_type: str) -> pl.DataFrame: + org_cols = [ + c + for c in schema.names() + if c.startswith(f"{org_type}_") + and c not in (f"{org_type}_latitude", f"{org_type}_longitude") + ] + select_list = ", ".join(org_cols) + group_list = ", ".join(org_cols) + sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}' + return get_cursor().execute(sql).pl() + + +DF_ACHETEURS = build_org_frame("acheteur") +DF_TITULAIRES = build_org_frame("titulaire") +DEPARTEMENTS = get_departements() +DEPARTEMENTS_GEOJSON = get_departements_geojson() +DATA_SCHEMA = get_data_schema() diff --git a/src/utils/search.py b/src/utils/search.py new file mode 100644 index 0000000..df998ad --- /dev/null +++ b/src/utils/search.py @@ -0,0 +1,84 @@ +import polars as pl +from unidecode import unidecode + +from src.utils.table import add_links +from src.utils.tracking import track_search + + +def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame: + """ + Search in either 'acheteur' or 'titulaire' DataFrame. + + :param dff: Polars DataFrame with acheteur or titulaire columns + :param query: User search string + :param org_type: 'acheteur' or 'titulaire' + :return: Filtered DataFrame with 'matches' column + """ + if not query.strip(): + return dff.select(pl.lit(False).alias("matches")) + + # Enregistrement des recherche dans Matomo + track_search(query, "home_page_search") + + # Normalize query + normalized_query = unidecode(query.strip()).upper() + tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()] + + # Define columns based on entity type + cols = [ + f"{org_type}_id", + f"{org_type}_nom", + f"{org_type}_departement_nom", + f"{org_type}_departement_code", + f"{org_type}_commune_nom", + ] + + # Concatenate all fields into one string per row + org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace( + "-", " " + ) + + # For each token, create a boolean column: True if token is found + token_matches = [] + for token in tokens: + token_match = org_str.str.contains(token).alias(f"token_{token}") + token_matches.append(token_match) + + # Count how many tokens match per row + match_score = pl.sum_horizontal(token_matches).alias("match_score") + + # For each token, create a boolean column: True if token is found + token_matches = [] + for token in tokens: + token_match = org_str.str.contains(token).alias(f"token_{token}") + token_matches.append(token_match) + + # Sélection des colonnes + if org_type == "acheteur": + dff = dff.select(cols + ["Marchés"]) + if org_type == "titulaire": + dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"]) + + # Apply and filter + dff = ( + dff.with_columns(token_matches + [match_score]) + .filter(pl.col("match_score") == len(tokens)) + .drop([f"token_{token}" for token in tokens]) + ) + + # Format result + dff = add_links(dff) + dff = dff.with_columns( + pl.concat_str( + pl.col(f"{org_type}_departement_nom"), + pl.lit(" ("), + pl.col(f"{org_type}_departement_code"), + pl.lit(")"), + ).alias("Département") + ) + + dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés") + dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum() + dff = dff.sort("Marchés", descending=True) + + return dff diff --git a/src/old_utils.py b/src/utils/table.py similarity index 52% rename from src/old_utils.py rename to src/utils/table.py index 190a8eb..a0814f7 100644 --- a/src/old_utils.py +++ b/src/utils/table.py @@ -1,31 +1,15 @@ -import json -import logging import os import uuid -from collections import OrderedDict -from datetime import datetime, timedelta -from time import localtime import polars as pl -import polars.selectors as cs from dash import no_update -from httpx import HTTPError, get, post -from unidecode import unidecode +from polars import selectors as cs -from src.db import conn as duckdb_conn # noqa: F401 (exposed for convenience) -from src.db import get_cursor, query_marches, schema # noqa: F401 - -logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(message)s", - level=logging.INFO, - datefmt="%Y-%m-%d %H:%M:%S", -) -logger = logging.getLogger("decp.info") -DEVELOPMENT = os.getenv("DEVELOPMENT", "False").lower() == "true" -if DEVELOPMENT: - logger.setLevel(logging.DEBUG) - -logging.getLogger("httpx").setLevel("WARNING") +from src.db import query_marches, schema +from src.utils import logger +from src.utils.data import DATA_SCHEMA +from src.utils.frontend import get_button_properties +from src.utils.tracking import track_search def split_filter_part(filter_part): @@ -222,55 +206,6 @@ def format_values(dff: pl.DataFrame) -> pl.DataFrame: return dff -def get_annuaire_data(siret: str) -> dict: - url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}" - try: - response = get(url).raise_for_status() - response = response.json()["results"][0] - except (HTTPError, IndexError): - response = None - logger.warning("Could not fetch data from recherche-entreprises.api.") - return response - - -def get_statistics() -> dict: - return ( - get( - "https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df", - follow_redirects=True, - ) - .raise_for_status() - .json() - ) - - -def get_departements() -> dict: - with open("data/departements.json", "rb") as f: - data = json.load(f) - return data - - -def get_departements_geojson() -> dict: - with open("./data/departements-1000m.geojson") as f: - geojson = json.load(f) - - # Ajout de feature.id - for f in geojson["features"]: - f["id"] = f["properties"]["code"] - - return geojson - - -def get_departement_region(code_postal): - if code_postal > "97000": - code_departement = code_postal[:3] - else: - code_departement = code_postal[:2] - nom_departement = DEPARTEMENTS[code_departement]["departement"] - nom_region = DEPARTEMENTS[code_departement]["region"] - return code_departement, nom_departement, nom_region - - def filter_table_data( lff: pl.LazyFrame, filter_query: str, filter_source: str ) -> pl.LazyFrame: @@ -433,129 +368,6 @@ def get_default_hidden_columns(page): return hidden_columns -def get_data_schema() -> dict: - # Récupération du schéma des données tabulaires - path = os.getenv("DATA_SCHEMA_PATH") - if path.startswith("http"): - original_schema: dict = get( - os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True - ).json() - elif os.path.exists(path): - with open(path) as f: - original_schema: dict = json.load(f) - else: - raise Exception(f"Chemin vers le schéma invalide: {path}") - - new_schema = OrderedDict() - - for col in original_schema["fields"]: - new_schema[col["name"]] = col - - return new_schema - - -def track_search(query, category): - if len(query) >= 4 and not DEVELOPMENT and os.getenv("MATOMO_DOMAIN"): - url = "https://decp.info" - params = { - "idsite": os.getenv("MATOMO_ID_SITE"), - "url": url, - "rec": "1", - "action_name": "search" if category == "home_page_search" else "filter", - "search_cat": category, - "rand": uuid.uuid4().hex, - "apiv": "1", - "h": localtime().tm_hour, - "m": localtime().tm_min, - "s": localtime().tm_sec, - "search": query, - "token_auth": os.getenv("MATOMO_TOKEN"), - } - post( - url=f"https://{os.getenv('MATOMO_DOMAIN')}/matomo.php", - params=params, - ).raise_for_status() - - -def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame: - """ - Search in either 'acheteur' or 'titulaire' DataFrame. - - :param dff: Polars DataFrame with acheteur or titulaire columns - :param query: User search string - :param org_type: 'acheteur' or 'titulaire' - :return: Filtered DataFrame with 'matches' column - """ - if not query.strip(): - return dff.select(pl.lit(False).alias("matches")) - - # Enregistrement des recherche dans Matomo - track_search(query, "home_page_search") - - # Normalize query - normalized_query = unidecode(query.strip()).upper() - tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()] - - # Define columns based on entity type - cols = [ - f"{org_type}_id", - f"{org_type}_nom", - f"{org_type}_departement_nom", - f"{org_type}_departement_code", - f"{org_type}_commune_nom", - ] - - # Concatenate all fields into one string per row - org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace( - "-", " " - ) - - # For each token, create a boolean column: True if token is found - token_matches = [] - for token in tokens: - token_match = org_str.str.contains(token).alias(f"token_{token}") - token_matches.append(token_match) - - # Count how many tokens match per row - match_score = pl.sum_horizontal(token_matches).alias("match_score") - - # For each token, create a boolean column: True if token is found - token_matches = [] - for token in tokens: - token_match = org_str.str.contains(token).alias(f"token_{token}") - token_matches.append(token_match) - - # Sélection des colonnes - if org_type == "acheteur": - dff = dff.select(cols + ["Marchés"]) - if org_type == "titulaire": - dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"]) - - # Apply and filter - dff = ( - dff.with_columns(token_matches + [match_score]) - .filter(pl.col("match_score") == len(tokens)) - .drop([f"token_{token}" for token in tokens]) - ) - - # Format result - dff = add_links(dff) - dff = dff.with_columns( - pl.concat_str( - pl.col(f"{org_type}_departement_nom"), - pl.lit(" ("), - pl.col(f"{org_type}_departement_code"), - pl.lit(")"), - ).alias("Département") - ) - - dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés") - dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum() - dff = dff.sort("Marchés", descending=True) - - return dff - - def prepare_table_data( data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table ): @@ -646,143 +458,6 @@ def prepare_table_data( ) -def prepare_dashboard_data( - lff: pl.LazyFrame, - dashboard_year=None, - dashboard_acheteur_id=None, - dashboard_acheteur_categorie=None, - dashboard_acheteur_departement_code=None, - dashboard_titulaire_id=None, - dashboard_titulaire_categorie=None, - dashboard_titulaire_departement_code=None, - dashboard_marche_type=None, - dashboard_marche_objet=None, - dashboard_marche_code_cpv=None, - dashboard_marche_considerations_sociales=None, - dashboard_marche_considerations_environnementales=None, - dashboard_marche_techniques=None, - dashboard_marche_innovant=None, - dashboard_marche_sous_traitance_declaree=None, - dashboard_montant_min=None, - dashboard_montant_max=None, -) -> pl.LazyFrame: - if dashboard_year: - lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year)) - else: - lff = lff.filter( - pl.col("dateNotification") > (datetime.now() - timedelta(days=365)) - ) - - if dashboard_acheteur_id: - lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id)) - else: - if dashboard_acheteur_categorie: - lff = lff.filter( - pl.col("acheteur_categorie") == dashboard_acheteur_categorie - ) - if dashboard_acheteur_departement_code: - lff = lff.filter( - pl.col("acheteur_departement_code").is_in( - dashboard_acheteur_departement_code - ) - ) - - if dashboard_titulaire_id: - lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id)) - else: - if dashboard_titulaire_categorie: - lff = lff.filter( - pl.col("titulaire_categorie") == dashboard_titulaire_categorie - ) - if dashboard_titulaire_departement_code: - lff = lff.filter( - pl.col("titulaire_departement_code").is_in( - dashboard_titulaire_departement_code - ) - ) - - if dashboard_marche_type: - lff = lff.filter(pl.col("type") == dashboard_marche_type) - - if dashboard_marche_objet: - lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}")) - - if dashboard_marche_code_cpv: - lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv)) - - if dashboard_marche_innovant and dashboard_marche_innovant != "all": - lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant) - - if ( - dashboard_marche_sous_traitance_declaree - and dashboard_marche_sous_traitance_declaree != "all" - ): - lff = lff.filter( - pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree - ) - - if dashboard_marche_techniques: - lff = lff.filter( - pl.col("techniques") - .str.split(", ") - .list.set_intersection(dashboard_marche_techniques) - .list.len() - > 0 - ) - - if dashboard_marche_considerations_sociales: - lff = lff.filter( - pl.col("considerationsSociales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_sociales) - .list.len() - > 0 - ) - - if dashboard_marche_considerations_environnementales: - lff = lff.filter( - pl.col("considerationsEnvironnementales") - .str.split(", ") - .list.set_intersection(dashboard_marche_considerations_environnementales) - .list.len() - > 0 - ) - - if dashboard_montant_min is not None: - lff = lff.filter(pl.col("montant") >= dashboard_montant_min) - - if dashboard_montant_max is not None: - lff = lff.filter(pl.col("montant") <= dashboard_montant_max) - - return lff - - -def get_button_properties(height): - if height > 65000: - download_disabled = True - download_text = "Téléchargement désactivé au-delà de 65 000 lignes" - download_title = " Ajoutez des filtres pour réduire le nombre de lignes, Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul." - elif height == 0: - download_disabled = True - download_text = "Pas de données à télécharger" - download_title = "" - else: - download_disabled = False - download_text = "Télécharger au format Excel" - download_title = "Télécharger les données telles qu'affichées au format Excel" - return download_disabled, download_text, download_title - - -def get_enum_values_as_dict(column_name): - try: - options = {} - for value in DATA_SCHEMA[column_name]["enum"]: - options[value] = value - return options - except KeyError: - return {"not_found": "not found"} - - def invert_columns(columns): """ Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa. @@ -797,79 +472,4 @@ def invert_columns(columns): return inverted_columns -def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dict: - org_types = {"acheteur": "GovernmentOrganization", "titulaire": "Organization"} - address = None - if type_org_id.lower() == "siret" and len(org_id) == 14: - annuaire_data = get_annuaire_data(org_id) - annuaire_address = annuaire_data["matching_etablissements"][0] - code_postal = annuaire_address["code_postal"] - commune = annuaire_address["libelle_commune"] - - address = ( - { - "@type": "PostalAddress", - "streetAddress": annuaire_address.get("adresse", "") - .replace(code_postal, "") - .replace(commune, "") - .strip(), - "addressLocality": commune, - "postalCode": code_postal, - "addressCountry": "FR", - }, - ) - - jsonld = { - "@type": org_types[org_type], - "name": org_name, - "url": f"https://decp.info/{org_type}s/{org_id}", - "sameAs": f"https://annuaire-entreprises.data.gouv.fr/etablissement/{org_id}", - "identifier": { - "@type": "PropertyValue", - "propertyID": type_org_id.lower(), - "value": org_id, - }, - } - - if address: - jsonld["address"] = address - - return jsonld - - -# df_acheteurs / df_titulaires sont conservés en mémoire pour alimenter -# la recherche sur la page d'accueil (autocomplétion, filtrage par sous-chaîne -# à chaque frappe). Les colonnes reproduisent la sortie historique de -# get_org_data(df, org_type). -def _build_org_frame(org_type: str) -> pl.DataFrame: - org_cols = [ - c - for c in schema.names() - if c.startswith(f"{org_type}_") - and c not in (f"{org_type}_latitude", f"{org_type}_longitude") - ] - select_list = ", ".join(org_cols) - group_list = ", ".join(org_cols) - sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}' - return get_cursor().execute(sql).pl() - - -DF_ACHETEURS = _build_org_frame("acheteur") -DF_TITULAIRES = _build_org_frame("titulaire") - COLUMNS = schema.names() - -DEPARTEMENTS = get_departements() -DEPARTEMENTS_GEOJSON = get_departements_geojson() -DOMAIN_NAME = ( - "test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info" -) -META_CONTENT = { - "image_url": f"https://{DOMAIN_NAME}/assets/decp.info.png", - "title": "decp.info - exploration des marchés publics français", - "description": ( - "Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. " - "Pour une commande publique accessible à toutes et tous." - ), -} -DATA_SCHEMA = get_data_schema()