colin/colibre
1
0
Fork 0
Code Issues 30 Pull Requests Actions Packages Projects Releases 33 Activity

Refactorisation des fonctions utils

Browse Source
This commit is contained in:
Colin Maudry
2026-04-18 18:32:57 +02:00
parent e352624c02
commit e4e1438220
14 changed files with 333 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/figures.py
+2 -7
View File
@@ -13,13 +13,8 @@ from dash import dash_table, dcc, html
from dash_extensions.javascript import Namespace
from src.db import schema
from src.old_utils import (
DATA_SCHEMA,
DEPARTEMENTS_GEOJSON,
add_links,
format_number,
setup_table_columns,
)
from src.utils.data import DATA_SCHEMA, DEPARTEMENTS_GEOJSON
from src.utils.table import add_links, format_number, setup_table_columns
def get_yearly_statistics(statistics, today_str) -> html.Div:
src/pages/a-propos.py
+1 -1
View File
@@ -3,7 +3,7 @@ import os
from dash import dcc, html, register_page
from src.figures import get_sources_tables
from src.old_utils import META_CONTENT
from src.utils.seo import META_CONTENT
NAME = "À propos"
src/pages/acheteur.py
+4 -6
View File
@@ -24,16 +24,14 @@ from src.figures import (
make_column_picker,
point_on_map,
)
from src.old_utils import (
from src.utils.data import DF_ACHETEURS, get_annuaire_data, get_departement_region
from src.utils.frontend import get_button_properties
from src.utils.seo import META_CONTENT
from src.utils.table import (
COLUMNS,
DF_ACHETEURS,
META_CONTENT,
filter_table_data,
format_number,
get_annuaire_data,
get_button_properties,
get_default_hidden_columns,
get_departement_region,
prepare_table_data,
sort_table_data,
)
src/pages/arbre/departement.py
+1 -1
View File
@@ -1,7 +1,7 @@
from dash import Input, Output, callback, dcc, html, register_page
from src.db import get_cursor
from src.old_utils import DEPARTEMENTS
from src.utils.data import DEPARTEMENTS
NAME = "Département"
src/pages/arbre/departements.py
+1 -1
View File
@@ -1,6 +1,6 @@
from dash import dcc, html, register_page
from src.old_utils import DEPARTEMENTS
from src.utils.data import DEPARTEMENTS
NAME = "Départements"
src/pages/arbre/liste_marches_org.py
+1 -1
View File
@@ -2,7 +2,7 @@ import polars as pl
from dash import Input, Output, callback, dcc, html, register_page
from src.db import get_cursor
from src.old_utils import DF_ACHETEURS, DF_TITULAIRES
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
NAME = "Liste des marchés publics"
src/pages/marche.py
+3 -7
View File
@@ -6,13 +6,9 @@ from dash import Input, Output, callback, dcc, html, register_page
from polars import selectors as cs
from src.db import query_marches
from src.old_utils import (
DATA_SCHEMA,
META_CONTENT,
format_values,
make_org_jsonld,
unformat_montant,
)
from src.utils.data import DATA_SCHEMA
from src.utils.seo import META_CONTENT, make_org_jsonld
from src.utils.table import format_values, unformat_montant
def get_title(uid: str = None) -> str:
src/pages/observatoire.py
+5 -7
View File
@@ -30,18 +30,16 @@ from src.figures import (
make_column_picker,
make_donut,
)
from src.old_utils import (
COLUMNS,
from src.utils import logger
from src.utils.data import (
DEPARTEMENTS,
DF_ACHETEURS,
DF_TITULAIRES,
META_CONTENT,
get_default_hidden_columns,
get_enum_values_as_dict,
logger,
prepare_dashboard_data,
prepare_table_data,
)
from src.utils.frontend import get_enum_values_as_dict
from src.utils.seo import META_CONTENT
from src.utils.table import COLUMNS, get_default_hidden_columns, prepare_table_data
NAME = "Observatoire"
src/pages/recherche.py
+4 -7
View File
@@ -2,13 +2,10 @@ import dash_bootstrap_components as dbc
from dash import Input, Output, State, callback, dcc, html, register_page
from src.figures import DataTable
from src.old_utils import (
DF_ACHETEURS,
DF_TITULAIRES,
META_CONTENT,
search_org,
setup_table_columns,
)
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
from src.utils.search import search_org
from src.utils.seo import META_CONTENT
from src.utils.table import setup_table_columns
NAME = "Recherche"
src/pages/tableau.py
+3 -3
View File
@@ -21,13 +21,13 @@ from dash import (
from src.db import query_marches, schema
from src.figures import DataTable, make_column_picker
from src.old_utils import (
from src.utils import logger
from src.utils.seo import META_CONTENT
from src.utils.table import (
COLUMNS,
META_CONTENT,
filter_table_data,
get_default_hidden_columns,
invert_columns,
logger,
prepare_table_data,
sort_table_data,
)
src/pages/titulaire.py
+4 -6
View File
@@ -23,16 +23,14 @@ from src.figures import (
make_column_picker,
point_on_map,
)
from src.old_utils import (
from src.utils.data import DF_TITULAIRES, get_annuaire_data, get_departement_region
from src.utils.frontend import get_button_properties
from src.utils.seo import META_CONTENT
from src.utils.table import (
COLUMNS,
DF_TITULAIRES,
META_CONTENT,
filter_table_data,
format_number,
get_annuaire_data,
get_button_properties,
get_default_hidden_columns,
get_departement_region,
prepare_table_data,
sort_table_data,
)
src/utils/data.py
+214
View File
@@ -0,0 +1,214 @@
import json
import logging
import os
from collections import OrderedDict
from datetime import datetime, timedelta
import polars as pl
from httpx import HTTPError, get
from src.db import get_cursor, schema
from src.utils import logger
logging.getLogger("httpx").setLevel("WARNING")
def get_annuaire_data(siret: str) -> dict:
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
try:
response = get(url).raise_for_status()
response = response.json()["results"][0]
except (HTTPError, IndexError):
response = None
logger.warning("Could not fetch data from recherche-entreprises.api.")
return response
def get_statistics() -> dict:
return (
get(
"https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df",
follow_redirects=True,
)
.raise_for_status()
.json()
)
def get_departements() -> dict:
with open("data/departements.json", "rb") as f:
data = json.load(f)
return data
def get_departements_geojson() -> dict:
with open("./data/departements-1000m.geojson") as f:
geojson = json.load(f)
# Ajout de feature.id
for f in geojson["features"]:
f["id"] = f["properties"]["code"]
return geojson
def get_departement_region(code_postal):
if code_postal > "97000":
code_departement = code_postal[:3]
else:
code_departement = code_postal[:2]
nom_departement = DEPARTEMENTS[code_departement]["departement"]
nom_region = DEPARTEMENTS[code_departement]["region"]
return code_departement, nom_departement, nom_region
def get_data_schema() -> dict:
# Récupération du schéma des données tabulaires
path = os.getenv("DATA_SCHEMA_PATH")
if path.startswith("http"):
original_schema: dict = get(
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
).json()
elif os.path.exists(path):
with open(path) as f:
original_schema: dict = json.load(f)
else:
raise Exception(f"Chemin vers le schéma invalide: {path}")
new_schema = OrderedDict()
for col in original_schema["fields"]:
new_schema[col["name"]] = col
return new_schema
def prepare_dashboard_data(
lff: pl.LazyFrame,
dashboard_year=None,
dashboard_acheteur_id=None,
dashboard_acheteur_categorie=None,
dashboard_acheteur_departement_code=None,
dashboard_titulaire_id=None,
dashboard_titulaire_categorie=None,
dashboard_titulaire_departement_code=None,
dashboard_marche_type=None,
dashboard_marche_objet=None,
dashboard_marche_code_cpv=None,
dashboard_marche_considerations_sociales=None,
dashboard_marche_considerations_environnementales=None,
dashboard_marche_techniques=None,
dashboard_marche_innovant=None,
dashboard_marche_sous_traitance_declaree=None,
dashboard_montant_min=None,
dashboard_montant_max=None,
) -> pl.LazyFrame:
if dashboard_year:
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
else:
lff = lff.filter(
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
)
if dashboard_acheteur_id:
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
else:
if dashboard_acheteur_categorie:
lff = lff.filter(
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
)
if dashboard_acheteur_departement_code:
lff = lff.filter(
pl.col("acheteur_departement_code").is_in(
dashboard_acheteur_departement_code
)
)
if dashboard_titulaire_id:
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
else:
if dashboard_titulaire_categorie:
lff = lff.filter(
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
)
if dashboard_titulaire_departement_code:
lff = lff.filter(
pl.col("titulaire_departement_code").is_in(
dashboard_titulaire_departement_code
)
)
if dashboard_marche_type:
lff = lff.filter(pl.col("type") == dashboard_marche_type)
if dashboard_marche_objet:
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
if dashboard_marche_code_cpv:
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
if (
dashboard_marche_sous_traitance_declaree
and dashboard_marche_sous_traitance_declaree != "all"
):
lff = lff.filter(
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
)
if dashboard_marche_techniques:
lff = lff.filter(
pl.col("techniques")
.str.split(", ")
.list.set_intersection(dashboard_marche_techniques)
.list.len()
> 0
)
if dashboard_marche_considerations_sociales:
lff = lff.filter(
pl.col("considerationsSociales")
.str.split(", ")
.list.set_intersection(dashboard_marche_considerations_sociales)
.list.len()
> 0
)
if dashboard_marche_considerations_environnementales:
lff = lff.filter(
pl.col("considerationsEnvironnementales")
.str.split(", ")
.list.set_intersection(dashboard_marche_considerations_environnementales)
.list.len()
> 0
)
if dashboard_montant_min is not None:
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
if dashboard_montant_max is not None:
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
return lff
def build_org_frame(org_type: str) -> pl.DataFrame:
org_cols = [
c
for c in schema.names()
if c.startswith(f"{org_type}_")
and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
]
select_list = ", ".join(org_cols)
group_list = ", ".join(org_cols)
sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
return get_cursor().execute(sql).pl()
DF_ACHETEURS = build_org_frame("acheteur")
DF_TITULAIRES = build_org_frame("titulaire")
DEPARTEMENTS = get_departements()
DEPARTEMENTS_GEOJSON = get_departements_geojson()
DATA_SCHEMA = get_data_schema()
src/utils/search.py
+84
View File
@@ -0,0 +1,84 @@
import polars as pl
from unidecode import unidecode
from src.utils.table import add_links
from src.utils.tracking import track_search
def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
"""
Search in either 'acheteur' or 'titulaire' DataFrame.
:param dff: Polars DataFrame with acheteur or titulaire columns
:param query: User search string
:param org_type: 'acheteur' or 'titulaire'
:return: Filtered DataFrame with 'matches' column
"""
if not query.strip():
return dff.select(pl.lit(False).alias("matches"))
# Enregistrement des recherche dans Matomo
track_search(query, "home_page_search")
# Normalize query
normalized_query = unidecode(query.strip()).upper()
tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()]
# Define columns based on entity type
cols = [
f"{org_type}_id",
f"{org_type}_nom",
f"{org_type}_departement_nom",
f"{org_type}_departement_code",
f"{org_type}_commune_nom",
]
# Concatenate all fields into one string per row
org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace(
"-", " "
)
# For each token, create a boolean column: True if token is found
token_matches = []
for token in tokens:
token_match = org_str.str.contains(token).alias(f"token_{token}")
token_matches.append(token_match)
# Count how many tokens match per row
match_score = pl.sum_horizontal(token_matches).alias("match_score")
# For each token, create a boolean column: True if token is found
token_matches = []
for token in tokens:
token_match = org_str.str.contains(token).alias(f"token_{token}")
token_matches.append(token_match)
# Sélection des colonnes
if org_type == "acheteur":
dff = dff.select(cols + ["Marchés"])
if org_type == "titulaire":
dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"])
# Apply and filter
dff = (
dff.with_columns(token_matches + [match_score])
.filter(pl.col("match_score") == len(tokens))
.drop([f"token_{token}" for token in tokens])
)
# Format result
dff = add_links(dff)
dff = dff.with_columns(
pl.concat_str(
pl.col(f"{org_type}_departement_nom"),
pl.lit(" ("),
pl.col(f"{org_type}_departement_code"),
pl.lit(")"),
).alias("Département")
)
dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés")
dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum()
dff = dff.sort("Marchés", descending=True)
return dff
src/old_utils.py → src/utils/table.py
+6 -406
View File
@@ -1,31 +1,15 @@
import json
import logging
import os
import uuid
from collections import OrderedDict
from datetime import datetime, timedelta
from time import localtime
import polars as pl
import polars.selectors as cs
from dash import no_update
from httpx import HTTPError, get, post
from unidecode import unidecode
from polars import selectors as cs
from src.db import conn as duckdb_conn # noqa: F401 (exposed for convenience)
from src.db import get_cursor, query_marches, schema # noqa: F401
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
level=logging.INFO,
datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("decp.info")
DEVELOPMENT = os.getenv("DEVELOPMENT", "False").lower() == "true"
if DEVELOPMENT:
logger.setLevel(logging.DEBUG)
logging.getLogger("httpx").setLevel("WARNING")
from src.db import query_marches, schema
from src.utils import logger
from src.utils.data import DATA_SCHEMA
from src.utils.frontend import get_button_properties
from src.utils.tracking import track_search
def split_filter_part(filter_part):
@@ -222,55 +206,6 @@ def format_values(dff: pl.DataFrame) -> pl.DataFrame:
return dff
def get_annuaire_data(siret: str) -> dict:
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
try:
response = get(url).raise_for_status()
response = response.json()["results"][0]
except (HTTPError, IndexError):
response = None
logger.warning("Could not fetch data from recherche-entreprises.api.")
return response
def get_statistics() -> dict:
return (
get(
"https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df",
follow_redirects=True,
)
.raise_for_status()
.json()
)
def get_departements() -> dict:
with open("data/departements.json", "rb") as f:
data = json.load(f)
return data
def get_departements_geojson() -> dict:
with open("./data/departements-1000m.geojson") as f:
geojson = json.load(f)
# Ajout de feature.id
for f in geojson["features"]:
f["id"] = f["properties"]["code"]
return geojson
def get_departement_region(code_postal):
if code_postal > "97000":
code_departement = code_postal[:3]
else:
code_departement = code_postal[:2]
nom_departement = DEPARTEMENTS[code_departement]["departement"]
nom_region = DEPARTEMENTS[code_departement]["region"]
return code_departement, nom_departement, nom_region
def filter_table_data(
lff: pl.LazyFrame, filter_query: str, filter_source: str
) -> pl.LazyFrame:
@@ -433,129 +368,6 @@ def get_default_hidden_columns(page):
return hidden_columns
def get_data_schema() -> dict:
# Récupération du schéma des données tabulaires
path = os.getenv("DATA_SCHEMA_PATH")
if path.startswith("http"):
original_schema: dict = get(
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
).json()
elif os.path.exists(path):
with open(path) as f:
original_schema: dict = json.load(f)
else:
raise Exception(f"Chemin vers le schéma invalide: {path}")
new_schema = OrderedDict()
for col in original_schema["fields"]:
new_schema[col["name"]] = col
return new_schema
def track_search(query, category):
if len(query) >= 4 and not DEVELOPMENT and os.getenv("MATOMO_DOMAIN"):
url = "https://decp.info"
params = {
"idsite": os.getenv("MATOMO_ID_SITE"),
"url": url,
"rec": "1",
"action_name": "search" if category == "home_page_search" else "filter",
"search_cat": category,
"rand": uuid.uuid4().hex,
"apiv": "1",
"h": localtime().tm_hour,
"m": localtime().tm_min,
"s": localtime().tm_sec,
"search": query,
"token_auth": os.getenv("MATOMO_TOKEN"),
}
post(
url=f"https://{os.getenv('MATOMO_DOMAIN')}/matomo.php",
params=params,
).raise_for_status()
def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
"""
Search in either 'acheteur' or 'titulaire' DataFrame.
:param dff: Polars DataFrame with acheteur or titulaire columns
:param query: User search string
:param org_type: 'acheteur' or 'titulaire'
:return: Filtered DataFrame with 'matches' column
"""
if not query.strip():
return dff.select(pl.lit(False).alias("matches"))
# Enregistrement des recherche dans Matomo
track_search(query, "home_page_search")
# Normalize query
normalized_query = unidecode(query.strip()).upper()
tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()]
# Define columns based on entity type
cols = [
f"{org_type}_id",
f"{org_type}_nom",
f"{org_type}_departement_nom",
f"{org_type}_departement_code",
f"{org_type}_commune_nom",
]
# Concatenate all fields into one string per row
org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace(
"-", " "
)
# For each token, create a boolean column: True if token is found
token_matches = []
for token in tokens:
token_match = org_str.str.contains(token).alias(f"token_{token}")
token_matches.append(token_match)
# Count how many tokens match per row
match_score = pl.sum_horizontal(token_matches).alias("match_score")
# For each token, create a boolean column: True if token is found
token_matches = []
for token in tokens:
token_match = org_str.str.contains(token).alias(f"token_{token}")
token_matches.append(token_match)
# Sélection des colonnes
if org_type == "acheteur":
dff = dff.select(cols + ["Marchés"])
if org_type == "titulaire":
dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"])
# Apply and filter
dff = (
dff.with_columns(token_matches + [match_score])
.filter(pl.col("match_score") == len(tokens))
.drop([f"token_{token}" for token in tokens])
)
# Format result
dff = add_links(dff)
dff = dff.with_columns(
pl.concat_str(
pl.col(f"{org_type}_departement_nom"),
pl.lit(" ("),
pl.col(f"{org_type}_departement_code"),
pl.lit(")"),
).alias("Département")
)
dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés")
dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum()
dff = dff.sort("Marchés", descending=True)
return dff
def prepare_table_data(
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
):
@@ -646,143 +458,6 @@ def prepare_table_data(
)
def prepare_dashboard_data(
lff: pl.LazyFrame,
dashboard_year=None,
dashboard_acheteur_id=None,
dashboard_acheteur_categorie=None,
dashboard_acheteur_departement_code=None,
dashboard_titulaire_id=None,
dashboard_titulaire_categorie=None,
dashboard_titulaire_departement_code=None,
dashboard_marche_type=None,
dashboard_marche_objet=None,
dashboard_marche_code_cpv=None,
dashboard_marche_considerations_sociales=None,
dashboard_marche_considerations_environnementales=None,
dashboard_marche_techniques=None,
dashboard_marche_innovant=None,
dashboard_marche_sous_traitance_declaree=None,
dashboard_montant_min=None,
dashboard_montant_max=None,
) -> pl.LazyFrame:
if dashboard_year:
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
else:
lff = lff.filter(
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
)
if dashboard_acheteur_id:
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
else:
if dashboard_acheteur_categorie:
lff = lff.filter(
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
)
if dashboard_acheteur_departement_code:
lff = lff.filter(
pl.col("acheteur_departement_code").is_in(
dashboard_acheteur_departement_code
)
)
if dashboard_titulaire_id:
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
else:
if dashboard_titulaire_categorie:
lff = lff.filter(
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
)
if dashboard_titulaire_departement_code:
lff = lff.filter(
pl.col("titulaire_departement_code").is_in(
dashboard_titulaire_departement_code
)
)
if dashboard_marche_type:
lff = lff.filter(pl.col("type") == dashboard_marche_type)
if dashboard_marche_objet:
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
if dashboard_marche_code_cpv:
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
if (
dashboard_marche_sous_traitance_declaree
and dashboard_marche_sous_traitance_declaree != "all"
):
lff = lff.filter(
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
)
if dashboard_marche_techniques:
lff = lff.filter(
pl.col("techniques")
.str.split(", ")
.list.set_intersection(dashboard_marche_techniques)
.list.len()
> 0
)
if dashboard_marche_considerations_sociales:
lff = lff.filter(
pl.col("considerationsSociales")
.str.split(", ")
.list.set_intersection(dashboard_marche_considerations_sociales)
.list.len()
> 0
)
if dashboard_marche_considerations_environnementales:
lff = lff.filter(
pl.col("considerationsEnvironnementales")
.str.split(", ")
.list.set_intersection(dashboard_marche_considerations_environnementales)
.list.len()
> 0
)
if dashboard_montant_min is not None:
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
if dashboard_montant_max is not None:
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
return lff
def get_button_properties(height):
if height > 65000:
download_disabled = True
download_text = "Téléchargement désactivé au-delà de 65 000 lignes"
download_title = " Ajoutez des filtres pour réduire le nombre de lignes, Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul."
elif height == 0:
download_disabled = True
download_text = "Pas de données à télécharger"
download_title = ""
else:
download_disabled = False
download_text = "Télécharger au format Excel"
download_title = "Télécharger les données telles qu'affichées au format Excel"
return download_disabled, download_text, download_title
def get_enum_values_as_dict(column_name):
try:
options = {}
for value in DATA_SCHEMA[column_name]["enum"]:
options[value] = value
return options
except KeyError:
return {"not_found": "not found"}
def invert_columns(columns):
"""
Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa.
@@ -797,79 +472,4 @@ def invert_columns(columns):
return inverted_columns
def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dict:
org_types = {"acheteur": "GovernmentOrganization", "titulaire": "Organization"}
address = None
if type_org_id.lower() == "siret" and len(org_id) == 14:
annuaire_data = get_annuaire_data(org_id)
annuaire_address = annuaire_data["matching_etablissements"][0]
code_postal = annuaire_address["code_postal"]
commune = annuaire_address["libelle_commune"]
address = (
{
"@type": "PostalAddress",
"streetAddress": annuaire_address.get("adresse", "")
.replace(code_postal, "")
.replace(commune, "")
.strip(),
"addressLocality": commune,
"postalCode": code_postal,
"addressCountry": "FR",
},
)
jsonld = {
"@type": org_types[org_type],
"name": org_name,
"url": f"https://decp.info/{org_type}s/{org_id}",
"sameAs": f"https://annuaire-entreprises.data.gouv.fr/etablissement/{org_id}",
"identifier": {
"@type": "PropertyValue",
"propertyID": type_org_id.lower(),
"value": org_id,
},
}
if address:
jsonld["address"] = address
return jsonld
# df_acheteurs / df_titulaires sont conservés en mémoire pour alimenter
# la recherche sur la page d'accueil (autocomplétion, filtrage par sous-chaîne
# à chaque frappe). Les colonnes reproduisent la sortie historique de
# get_org_data(df, org_type).
def _build_org_frame(org_type: str) -> pl.DataFrame:
org_cols = [
c
for c in schema.names()
if c.startswith(f"{org_type}_")
and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
]
select_list = ", ".join(org_cols)
group_list = ", ".join(org_cols)
sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
return get_cursor().execute(sql).pl()
DF_ACHETEURS = _build_org_frame("acheteur")
DF_TITULAIRES = _build_org_frame("titulaire")
COLUMNS = schema.names()
DEPARTEMENTS = get_departements()
DEPARTEMENTS_GEOJSON = get_departements_geojson()
DOMAIN_NAME = (
"test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info"
)
META_CONTENT = {
"image_url": f"https://{DOMAIN_NAME}/assets/decp.info.png",
"title": "decp.info - exploration des marchés publics français",
"description": (
"Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. "
"Pour une commande publique accessible à toutes et tous."
),
}
DATA_SCHEMA = get_data_schema()