Refactorisation des fonctions utils
This commit is contained in:
Colin Maudry
+2
-7
@@ -13,13 +13,8 @@ from dash import dash_table, dcc, html
|
||||
from dash_extensions.javascript import Namespace
|
||||
|
||||
from src.db import schema
|
||||
from src.old_utils import (
|
||||
DATA_SCHEMA,
|
||||
DEPARTEMENTS_GEOJSON,
|
||||
add_links,
|
||||
format_number,
|
||||
setup_table_columns,
|
||||
)
|
||||
from src.utils.data import DATA_SCHEMA, DEPARTEMENTS_GEOJSON
|
||||
from src.utils.table import add_links, format_number, setup_table_columns
|
||||
|
||||
|
||||
def get_yearly_statistics(statistics, today_str) -> html.Div:
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
from dash import dcc, html, register_page
|
||||
|
||||
from src.figures import get_sources_tables
|
||||
from src.old_utils import META_CONTENT
|
||||
from src.utils.seo import META_CONTENT
|
||||
|
||||
NAME = "À propos"
|
||||
|
||||
|
||||
@@ -24,16 +24,14 @@ from src.figures import (
|
||||
make_column_picker,
|
||||
point_on_map,
|
||||
)
|
||||
from src.old_utils import (
|
||||
from src.utils.data import DF_ACHETEURS, get_annuaire_data, get_departement_region
|
||||
from src.utils.frontend import get_button_properties
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import (
|
||||
COLUMNS,
|
||||
DF_ACHETEURS,
|
||||
META_CONTENT,
|
||||
filter_table_data,
|
||||
format_number,
|
||||
get_annuaire_data,
|
||||
get_button_properties,
|
||||
get_default_hidden_columns,
|
||||
get_departement_region,
|
||||
prepare_table_data,
|
||||
sort_table_data,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dash import Input, Output, callback, dcc, html, register_page
|
||||
|
||||
from src.db import get_cursor
|
||||
from src.old_utils import DEPARTEMENTS
|
||||
from src.utils.data import DEPARTEMENTS
|
||||
|
||||
NAME = "Département"
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from dash import dcc, html, register_page
|
||||
|
||||
from src.old_utils import DEPARTEMENTS
|
||||
from src.utils.data import DEPARTEMENTS
|
||||
|
||||
NAME = "Départements"
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import polars as pl
|
||||
from dash import Input, Output, callback, dcc, html, register_page
|
||||
|
||||
from src.db import get_cursor
|
||||
from src.old_utils import DF_ACHETEURS, DF_TITULAIRES
|
||||
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
|
||||
|
||||
NAME = "Liste des marchés publics"
|
||||
|
||||
|
||||
+3
-7
@@ -6,13 +6,9 @@ from dash import Input, Output, callback, dcc, html, register_page
|
||||
from polars import selectors as cs
|
||||
|
||||
from src.db import query_marches
|
||||
from src.old_utils import (
|
||||
DATA_SCHEMA,
|
||||
META_CONTENT,
|
||||
format_values,
|
||||
make_org_jsonld,
|
||||
unformat_montant,
|
||||
)
|
||||
from src.utils.data import DATA_SCHEMA
|
||||
from src.utils.seo import META_CONTENT, make_org_jsonld
|
||||
from src.utils.table import format_values, unformat_montant
|
||||
|
||||
|
||||
def get_title(uid: str = None) -> str:
|
||||
|
||||
@@ -30,18 +30,16 @@ from src.figures import (
|
||||
make_column_picker,
|
||||
make_donut,
|
||||
)
|
||||
from src.old_utils import (
|
||||
COLUMNS,
|
||||
from src.utils import logger
|
||||
from src.utils.data import (
|
||||
DEPARTEMENTS,
|
||||
DF_ACHETEURS,
|
||||
DF_TITULAIRES,
|
||||
META_CONTENT,
|
||||
get_default_hidden_columns,
|
||||
get_enum_values_as_dict,
|
||||
logger,
|
||||
prepare_dashboard_data,
|
||||
prepare_table_data,
|
||||
)
|
||||
from src.utils.frontend import get_enum_values_as_dict
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import COLUMNS, get_default_hidden_columns, prepare_table_data
|
||||
|
||||
NAME = "Observatoire"
|
||||
|
||||
|
||||
@@ -2,13 +2,10 @@ import dash_bootstrap_components as dbc
|
||||
from dash import Input, Output, State, callback, dcc, html, register_page
|
||||
|
||||
from src.figures import DataTable
|
||||
from src.old_utils import (
|
||||
DF_ACHETEURS,
|
||||
DF_TITULAIRES,
|
||||
META_CONTENT,
|
||||
search_org,
|
||||
setup_table_columns,
|
||||
)
|
||||
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
|
||||
from src.utils.search import search_org
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import setup_table_columns
|
||||
|
||||
NAME = "Recherche"
|
||||
|
||||
|
||||
@@ -21,13 +21,13 @@ from dash import (
|
||||
|
||||
from src.db import query_marches, schema
|
||||
from src.figures import DataTable, make_column_picker
|
||||
from src.old_utils import (
|
||||
from src.utils import logger
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import (
|
||||
COLUMNS,
|
||||
META_CONTENT,
|
||||
filter_table_data,
|
||||
get_default_hidden_columns,
|
||||
invert_columns,
|
||||
logger,
|
||||
prepare_table_data,
|
||||
sort_table_data,
|
||||
)
|
||||
|
||||
@@ -23,16 +23,14 @@ from src.figures import (
|
||||
make_column_picker,
|
||||
point_on_map,
|
||||
)
|
||||
from src.old_utils import (
|
||||
from src.utils.data import DF_TITULAIRES, get_annuaire_data, get_departement_region
|
||||
from src.utils.frontend import get_button_properties
|
||||
from src.utils.seo import META_CONTENT
|
||||
from src.utils.table import (
|
||||
COLUMNS,
|
||||
DF_TITULAIRES,
|
||||
META_CONTENT,
|
||||
filter_table_data,
|
||||
format_number,
|
||||
get_annuaire_data,
|
||||
get_button_properties,
|
||||
get_default_hidden_columns,
|
||||
get_departement_region,
|
||||
prepare_table_data,
|
||||
sort_table_data,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
from httpx import HTTPError, get
|
||||
|
||||
from src.db import get_cursor, schema
|
||||
from src.utils import logger
|
||||
|
||||
logging.getLogger("httpx").setLevel("WARNING")
|
||||
|
||||
|
||||
def get_annuaire_data(siret: str) -> dict:
|
||||
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
|
||||
try:
|
||||
response = get(url).raise_for_status()
|
||||
response = response.json()["results"][0]
|
||||
except (HTTPError, IndexError):
|
||||
response = None
|
||||
logger.warning("Could not fetch data from recherche-entreprises.api.")
|
||||
return response
|
||||
|
||||
|
||||
def get_statistics() -> dict:
|
||||
return (
|
||||
get(
|
||||
"https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df",
|
||||
follow_redirects=True,
|
||||
)
|
||||
.raise_for_status()
|
||||
.json()
|
||||
)
|
||||
|
||||
|
||||
def get_departements() -> dict:
|
||||
with open("data/departements.json", "rb") as f:
|
||||
data = json.load(f)
|
||||
return data
|
||||
|
||||
|
||||
def get_departements_geojson() -> dict:
|
||||
with open("./data/departements-1000m.geojson") as f:
|
||||
geojson = json.load(f)
|
||||
|
||||
# Ajout de feature.id
|
||||
for f in geojson["features"]:
|
||||
f["id"] = f["properties"]["code"]
|
||||
|
||||
return geojson
|
||||
|
||||
|
||||
def get_departement_region(code_postal):
|
||||
if code_postal > "97000":
|
||||
code_departement = code_postal[:3]
|
||||
else:
|
||||
code_departement = code_postal[:2]
|
||||
nom_departement = DEPARTEMENTS[code_departement]["departement"]
|
||||
nom_region = DEPARTEMENTS[code_departement]["region"]
|
||||
return code_departement, nom_departement, nom_region
|
||||
|
||||
|
||||
def get_data_schema() -> dict:
|
||||
# Récupération du schéma des données tabulaires
|
||||
path = os.getenv("DATA_SCHEMA_PATH")
|
||||
if path.startswith("http"):
|
||||
original_schema: dict = get(
|
||||
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
|
||||
).json()
|
||||
elif os.path.exists(path):
|
||||
with open(path) as f:
|
||||
original_schema: dict = json.load(f)
|
||||
else:
|
||||
raise Exception(f"Chemin vers le schéma invalide: {path}")
|
||||
|
||||
new_schema = OrderedDict()
|
||||
|
||||
for col in original_schema["fields"]:
|
||||
new_schema[col["name"]] = col
|
||||
|
||||
return new_schema
|
||||
|
||||
|
||||
def prepare_dashboard_data(
|
||||
lff: pl.LazyFrame,
|
||||
dashboard_year=None,
|
||||
dashboard_acheteur_id=None,
|
||||
dashboard_acheteur_categorie=None,
|
||||
dashboard_acheteur_departement_code=None,
|
||||
dashboard_titulaire_id=None,
|
||||
dashboard_titulaire_categorie=None,
|
||||
dashboard_titulaire_departement_code=None,
|
||||
dashboard_marche_type=None,
|
||||
dashboard_marche_objet=None,
|
||||
dashboard_marche_code_cpv=None,
|
||||
dashboard_marche_considerations_sociales=None,
|
||||
dashboard_marche_considerations_environnementales=None,
|
||||
dashboard_marche_techniques=None,
|
||||
dashboard_marche_innovant=None,
|
||||
dashboard_marche_sous_traitance_declaree=None,
|
||||
dashboard_montant_min=None,
|
||||
dashboard_montant_max=None,
|
||||
) -> pl.LazyFrame:
|
||||
if dashboard_year:
|
||||
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
|
||||
else:
|
||||
lff = lff.filter(
|
||||
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
|
||||
)
|
||||
|
||||
if dashboard_acheteur_id:
|
||||
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
|
||||
else:
|
||||
if dashboard_acheteur_categorie:
|
||||
lff = lff.filter(
|
||||
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
|
||||
)
|
||||
if dashboard_acheteur_departement_code:
|
||||
lff = lff.filter(
|
||||
pl.col("acheteur_departement_code").is_in(
|
||||
dashboard_acheteur_departement_code
|
||||
)
|
||||
)
|
||||
|
||||
if dashboard_titulaire_id:
|
||||
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
|
||||
else:
|
||||
if dashboard_titulaire_categorie:
|
||||
lff = lff.filter(
|
||||
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
|
||||
)
|
||||
if dashboard_titulaire_departement_code:
|
||||
lff = lff.filter(
|
||||
pl.col("titulaire_departement_code").is_in(
|
||||
dashboard_titulaire_departement_code
|
||||
)
|
||||
)
|
||||
|
||||
if dashboard_marche_type:
|
||||
lff = lff.filter(pl.col("type") == dashboard_marche_type)
|
||||
|
||||
if dashboard_marche_objet:
|
||||
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
|
||||
|
||||
if dashboard_marche_code_cpv:
|
||||
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
|
||||
|
||||
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
|
||||
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
|
||||
|
||||
if (
|
||||
dashboard_marche_sous_traitance_declaree
|
||||
and dashboard_marche_sous_traitance_declaree != "all"
|
||||
):
|
||||
lff = lff.filter(
|
||||
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
|
||||
)
|
||||
|
||||
if dashboard_marche_techniques:
|
||||
lff = lff.filter(
|
||||
pl.col("techniques")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_techniques)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_marche_considerations_sociales:
|
||||
lff = lff.filter(
|
||||
pl.col("considerationsSociales")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_considerations_sociales)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_marche_considerations_environnementales:
|
||||
lff = lff.filter(
|
||||
pl.col("considerationsEnvironnementales")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_considerations_environnementales)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_montant_min is not None:
|
||||
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
|
||||
|
||||
if dashboard_montant_max is not None:
|
||||
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
|
||||
|
||||
return lff
|
||||
|
||||
|
||||
def build_org_frame(org_type: str) -> pl.DataFrame:
|
||||
org_cols = [
|
||||
c
|
||||
for c in schema.names()
|
||||
if c.startswith(f"{org_type}_")
|
||||
and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
|
||||
]
|
||||
select_list = ", ".join(org_cols)
|
||||
group_list = ", ".join(org_cols)
|
||||
sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
|
||||
return get_cursor().execute(sql).pl()
|
||||
|
||||
|
||||
DF_ACHETEURS = build_org_frame("acheteur")
|
||||
DF_TITULAIRES = build_org_frame("titulaire")
|
||||
DEPARTEMENTS = get_departements()
|
||||
DEPARTEMENTS_GEOJSON = get_departements_geojson()
|
||||
DATA_SCHEMA = get_data_schema()
|
||||
@@ -0,0 +1,84 @@
|
||||
import polars as pl
|
||||
from unidecode import unidecode
|
||||
|
||||
from src.utils.table import add_links
|
||||
from src.utils.tracking import track_search
|
||||
|
||||
|
||||
def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
|
||||
"""
|
||||
Search in either 'acheteur' or 'titulaire' DataFrame.
|
||||
|
||||
:param dff: Polars DataFrame with acheteur or titulaire columns
|
||||
:param query: User search string
|
||||
:param org_type: 'acheteur' or 'titulaire'
|
||||
:return: Filtered DataFrame with 'matches' column
|
||||
"""
|
||||
if not query.strip():
|
||||
return dff.select(pl.lit(False).alias("matches"))
|
||||
|
||||
# Enregistrement des recherche dans Matomo
|
||||
track_search(query, "home_page_search")
|
||||
|
||||
# Normalize query
|
||||
normalized_query = unidecode(query.strip()).upper()
|
||||
tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()]
|
||||
|
||||
# Define columns based on entity type
|
||||
cols = [
|
||||
f"{org_type}_id",
|
||||
f"{org_type}_nom",
|
||||
f"{org_type}_departement_nom",
|
||||
f"{org_type}_departement_code",
|
||||
f"{org_type}_commune_nom",
|
||||
]
|
||||
|
||||
# Concatenate all fields into one string per row
|
||||
org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace(
|
||||
"-", " "
|
||||
)
|
||||
|
||||
# For each token, create a boolean column: True if token is found
|
||||
token_matches = []
|
||||
for token in tokens:
|
||||
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
||||
token_matches.append(token_match)
|
||||
|
||||
# Count how many tokens match per row
|
||||
match_score = pl.sum_horizontal(token_matches).alias("match_score")
|
||||
|
||||
# For each token, create a boolean column: True if token is found
|
||||
token_matches = []
|
||||
for token in tokens:
|
||||
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
||||
token_matches.append(token_match)
|
||||
|
||||
# Sélection des colonnes
|
||||
if org_type == "acheteur":
|
||||
dff = dff.select(cols + ["Marchés"])
|
||||
if org_type == "titulaire":
|
||||
dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"])
|
||||
|
||||
# Apply and filter
|
||||
dff = (
|
||||
dff.with_columns(token_matches + [match_score])
|
||||
.filter(pl.col("match_score") == len(tokens))
|
||||
.drop([f"token_{token}" for token in tokens])
|
||||
)
|
||||
|
||||
# Format result
|
||||
dff = add_links(dff)
|
||||
dff = dff.with_columns(
|
||||
pl.concat_str(
|
||||
pl.col(f"{org_type}_departement_nom"),
|
||||
pl.lit(" ("),
|
||||
pl.col(f"{org_type}_departement_code"),
|
||||
pl.lit(")"),
|
||||
).alias("Département")
|
||||
)
|
||||
|
||||
dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés")
|
||||
dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum()
|
||||
dff = dff.sort("Marchés", descending=True)
|
||||
|
||||
return dff
|
||||
@@ -1,31 +1,15 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta
|
||||
from time import localtime
|
||||
|
||||
import polars as pl
|
||||
import polars.selectors as cs
|
||||
from dash import no_update
|
||||
from httpx import HTTPError, get, post
|
||||
from unidecode import unidecode
|
||||
from polars import selectors as cs
|
||||
|
||||
from src.db import conn as duckdb_conn # noqa: F401 (exposed for convenience)
|
||||
from src.db import get_cursor, query_marches, schema # noqa: F401
|
||||
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s %(levelname)-8s %(message)s",
|
||||
level=logging.INFO,
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger("decp.info")
|
||||
DEVELOPMENT = os.getenv("DEVELOPMENT", "False").lower() == "true"
|
||||
if DEVELOPMENT:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
logging.getLogger("httpx").setLevel("WARNING")
|
||||
from src.db import query_marches, schema
|
||||
from src.utils import logger
|
||||
from src.utils.data import DATA_SCHEMA
|
||||
from src.utils.frontend import get_button_properties
|
||||
from src.utils.tracking import track_search
|
||||
|
||||
|
||||
def split_filter_part(filter_part):
|
||||
@@ -222,55 +206,6 @@ def format_values(dff: pl.DataFrame) -> pl.DataFrame:
|
||||
return dff
|
||||
|
||||
|
||||
def get_annuaire_data(siret: str) -> dict:
|
||||
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
|
||||
try:
|
||||
response = get(url).raise_for_status()
|
||||
response = response.json()["results"][0]
|
||||
except (HTTPError, IndexError):
|
||||
response = None
|
||||
logger.warning("Could not fetch data from recherche-entreprises.api.")
|
||||
return response
|
||||
|
||||
|
||||
def get_statistics() -> dict:
|
||||
return (
|
||||
get(
|
||||
"https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df",
|
||||
follow_redirects=True,
|
||||
)
|
||||
.raise_for_status()
|
||||
.json()
|
||||
)
|
||||
|
||||
|
||||
def get_departements() -> dict:
|
||||
with open("data/departements.json", "rb") as f:
|
||||
data = json.load(f)
|
||||
return data
|
||||
|
||||
|
||||
def get_departements_geojson() -> dict:
|
||||
with open("./data/departements-1000m.geojson") as f:
|
||||
geojson = json.load(f)
|
||||
|
||||
# Ajout de feature.id
|
||||
for f in geojson["features"]:
|
||||
f["id"] = f["properties"]["code"]
|
||||
|
||||
return geojson
|
||||
|
||||
|
||||
def get_departement_region(code_postal):
|
||||
if code_postal > "97000":
|
||||
code_departement = code_postal[:3]
|
||||
else:
|
||||
code_departement = code_postal[:2]
|
||||
nom_departement = DEPARTEMENTS[code_departement]["departement"]
|
||||
nom_region = DEPARTEMENTS[code_departement]["region"]
|
||||
return code_departement, nom_departement, nom_region
|
||||
|
||||
|
||||
def filter_table_data(
|
||||
lff: pl.LazyFrame, filter_query: str, filter_source: str
|
||||
) -> pl.LazyFrame:
|
||||
@@ -433,129 +368,6 @@ def get_default_hidden_columns(page):
|
||||
return hidden_columns
|
||||
|
||||
|
||||
def get_data_schema() -> dict:
|
||||
# Récupération du schéma des données tabulaires
|
||||
path = os.getenv("DATA_SCHEMA_PATH")
|
||||
if path.startswith("http"):
|
||||
original_schema: dict = get(
|
||||
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
|
||||
).json()
|
||||
elif os.path.exists(path):
|
||||
with open(path) as f:
|
||||
original_schema: dict = json.load(f)
|
||||
else:
|
||||
raise Exception(f"Chemin vers le schéma invalide: {path}")
|
||||
|
||||
new_schema = OrderedDict()
|
||||
|
||||
for col in original_schema["fields"]:
|
||||
new_schema[col["name"]] = col
|
||||
|
||||
return new_schema
|
||||
|
||||
|
||||
def track_search(query, category):
|
||||
if len(query) >= 4 and not DEVELOPMENT and os.getenv("MATOMO_DOMAIN"):
|
||||
url = "https://decp.info"
|
||||
params = {
|
||||
"idsite": os.getenv("MATOMO_ID_SITE"),
|
||||
"url": url,
|
||||
"rec": "1",
|
||||
"action_name": "search" if category == "home_page_search" else "filter",
|
||||
"search_cat": category,
|
||||
"rand": uuid.uuid4().hex,
|
||||
"apiv": "1",
|
||||
"h": localtime().tm_hour,
|
||||
"m": localtime().tm_min,
|
||||
"s": localtime().tm_sec,
|
||||
"search": query,
|
||||
"token_auth": os.getenv("MATOMO_TOKEN"),
|
||||
}
|
||||
post(
|
||||
url=f"https://{os.getenv('MATOMO_DOMAIN')}/matomo.php",
|
||||
params=params,
|
||||
).raise_for_status()
|
||||
|
||||
|
||||
def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
|
||||
"""
|
||||
Search in either 'acheteur' or 'titulaire' DataFrame.
|
||||
|
||||
:param dff: Polars DataFrame with acheteur or titulaire columns
|
||||
:param query: User search string
|
||||
:param org_type: 'acheteur' or 'titulaire'
|
||||
:return: Filtered DataFrame with 'matches' column
|
||||
"""
|
||||
if not query.strip():
|
||||
return dff.select(pl.lit(False).alias("matches"))
|
||||
|
||||
# Enregistrement des recherche dans Matomo
|
||||
track_search(query, "home_page_search")
|
||||
|
||||
# Normalize query
|
||||
normalized_query = unidecode(query.strip()).upper()
|
||||
tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()]
|
||||
|
||||
# Define columns based on entity type
|
||||
cols = [
|
||||
f"{org_type}_id",
|
||||
f"{org_type}_nom",
|
||||
f"{org_type}_departement_nom",
|
||||
f"{org_type}_departement_code",
|
||||
f"{org_type}_commune_nom",
|
||||
]
|
||||
|
||||
# Concatenate all fields into one string per row
|
||||
org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace(
|
||||
"-", " "
|
||||
)
|
||||
|
||||
# For each token, create a boolean column: True if token is found
|
||||
token_matches = []
|
||||
for token in tokens:
|
||||
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
||||
token_matches.append(token_match)
|
||||
|
||||
# Count how many tokens match per row
|
||||
match_score = pl.sum_horizontal(token_matches).alias("match_score")
|
||||
|
||||
# For each token, create a boolean column: True if token is found
|
||||
token_matches = []
|
||||
for token in tokens:
|
||||
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
||||
token_matches.append(token_match)
|
||||
|
||||
# Sélection des colonnes
|
||||
if org_type == "acheteur":
|
||||
dff = dff.select(cols + ["Marchés"])
|
||||
if org_type == "titulaire":
|
||||
dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"])
|
||||
|
||||
# Apply and filter
|
||||
dff = (
|
||||
dff.with_columns(token_matches + [match_score])
|
||||
.filter(pl.col("match_score") == len(tokens))
|
||||
.drop([f"token_{token}" for token in tokens])
|
||||
)
|
||||
|
||||
# Format result
|
||||
dff = add_links(dff)
|
||||
dff = dff.with_columns(
|
||||
pl.concat_str(
|
||||
pl.col(f"{org_type}_departement_nom"),
|
||||
pl.lit(" ("),
|
||||
pl.col(f"{org_type}_departement_code"),
|
||||
pl.lit(")"),
|
||||
).alias("Département")
|
||||
)
|
||||
|
||||
dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés")
|
||||
dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum()
|
||||
dff = dff.sort("Marchés", descending=True)
|
||||
|
||||
return dff
|
||||
|
||||
|
||||
def prepare_table_data(
|
||||
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
|
||||
):
|
||||
@@ -646,143 +458,6 @@ def prepare_table_data(
|
||||
)
|
||||
|
||||
|
||||
def prepare_dashboard_data(
|
||||
lff: pl.LazyFrame,
|
||||
dashboard_year=None,
|
||||
dashboard_acheteur_id=None,
|
||||
dashboard_acheteur_categorie=None,
|
||||
dashboard_acheteur_departement_code=None,
|
||||
dashboard_titulaire_id=None,
|
||||
dashboard_titulaire_categorie=None,
|
||||
dashboard_titulaire_departement_code=None,
|
||||
dashboard_marche_type=None,
|
||||
dashboard_marche_objet=None,
|
||||
dashboard_marche_code_cpv=None,
|
||||
dashboard_marche_considerations_sociales=None,
|
||||
dashboard_marche_considerations_environnementales=None,
|
||||
dashboard_marche_techniques=None,
|
||||
dashboard_marche_innovant=None,
|
||||
dashboard_marche_sous_traitance_declaree=None,
|
||||
dashboard_montant_min=None,
|
||||
dashboard_montant_max=None,
|
||||
) -> pl.LazyFrame:
|
||||
if dashboard_year:
|
||||
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
|
||||
else:
|
||||
lff = lff.filter(
|
||||
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
|
||||
)
|
||||
|
||||
if dashboard_acheteur_id:
|
||||
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
|
||||
else:
|
||||
if dashboard_acheteur_categorie:
|
||||
lff = lff.filter(
|
||||
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
|
||||
)
|
||||
if dashboard_acheteur_departement_code:
|
||||
lff = lff.filter(
|
||||
pl.col("acheteur_departement_code").is_in(
|
||||
dashboard_acheteur_departement_code
|
||||
)
|
||||
)
|
||||
|
||||
if dashboard_titulaire_id:
|
||||
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
|
||||
else:
|
||||
if dashboard_titulaire_categorie:
|
||||
lff = lff.filter(
|
||||
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
|
||||
)
|
||||
if dashboard_titulaire_departement_code:
|
||||
lff = lff.filter(
|
||||
pl.col("titulaire_departement_code").is_in(
|
||||
dashboard_titulaire_departement_code
|
||||
)
|
||||
)
|
||||
|
||||
if dashboard_marche_type:
|
||||
lff = lff.filter(pl.col("type") == dashboard_marche_type)
|
||||
|
||||
if dashboard_marche_objet:
|
||||
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
|
||||
|
||||
if dashboard_marche_code_cpv:
|
||||
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
|
||||
|
||||
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
|
||||
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
|
||||
|
||||
if (
|
||||
dashboard_marche_sous_traitance_declaree
|
||||
and dashboard_marche_sous_traitance_declaree != "all"
|
||||
):
|
||||
lff = lff.filter(
|
||||
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
|
||||
)
|
||||
|
||||
if dashboard_marche_techniques:
|
||||
lff = lff.filter(
|
||||
pl.col("techniques")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_techniques)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_marche_considerations_sociales:
|
||||
lff = lff.filter(
|
||||
pl.col("considerationsSociales")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_considerations_sociales)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_marche_considerations_environnementales:
|
||||
lff = lff.filter(
|
||||
pl.col("considerationsEnvironnementales")
|
||||
.str.split(", ")
|
||||
.list.set_intersection(dashboard_marche_considerations_environnementales)
|
||||
.list.len()
|
||||
> 0
|
||||
)
|
||||
|
||||
if dashboard_montant_min is not None:
|
||||
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
|
||||
|
||||
if dashboard_montant_max is not None:
|
||||
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
|
||||
|
||||
return lff
|
||||
|
||||
|
||||
def get_button_properties(height):
|
||||
if height > 65000:
|
||||
download_disabled = True
|
||||
download_text = "Téléchargement désactivé au-delà de 65 000 lignes"
|
||||
download_title = " Ajoutez des filtres pour réduire le nombre de lignes, Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul."
|
||||
elif height == 0:
|
||||
download_disabled = True
|
||||
download_text = "Pas de données à télécharger"
|
||||
download_title = ""
|
||||
else:
|
||||
download_disabled = False
|
||||
download_text = "Télécharger au format Excel"
|
||||
download_title = "Télécharger les données telles qu'affichées au format Excel"
|
||||
return download_disabled, download_text, download_title
|
||||
|
||||
|
||||
def get_enum_values_as_dict(column_name):
|
||||
try:
|
||||
options = {}
|
||||
for value in DATA_SCHEMA[column_name]["enum"]:
|
||||
options[value] = value
|
||||
return options
|
||||
except KeyError:
|
||||
return {"not_found": "not found"}
|
||||
|
||||
|
||||
def invert_columns(columns):
|
||||
"""
|
||||
Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa.
|
||||
@@ -797,79 +472,4 @@ def invert_columns(columns):
|
||||
return inverted_columns
|
||||
|
||||
|
||||
def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dict:
|
||||
org_types = {"acheteur": "GovernmentOrganization", "titulaire": "Organization"}
|
||||
address = None
|
||||
if type_org_id.lower() == "siret" and len(org_id) == 14:
|
||||
annuaire_data = get_annuaire_data(org_id)
|
||||
annuaire_address = annuaire_data["matching_etablissements"][0]
|
||||
code_postal = annuaire_address["code_postal"]
|
||||
commune = annuaire_address["libelle_commune"]
|
||||
|
||||
address = (
|
||||
{
|
||||
"@type": "PostalAddress",
|
||||
"streetAddress": annuaire_address.get("adresse", "")
|
||||
.replace(code_postal, "")
|
||||
.replace(commune, "")
|
||||
.strip(),
|
||||
"addressLocality": commune,
|
||||
"postalCode": code_postal,
|
||||
"addressCountry": "FR",
|
||||
},
|
||||
)
|
||||
|
||||
jsonld = {
|
||||
"@type": org_types[org_type],
|
||||
"name": org_name,
|
||||
"url": f"https://decp.info/{org_type}s/{org_id}",
|
||||
"sameAs": f"https://annuaire-entreprises.data.gouv.fr/etablissement/{org_id}",
|
||||
"identifier": {
|
||||
"@type": "PropertyValue",
|
||||
"propertyID": type_org_id.lower(),
|
||||
"value": org_id,
|
||||
},
|
||||
}
|
||||
|
||||
if address:
|
||||
jsonld["address"] = address
|
||||
|
||||
return jsonld
|
||||
|
||||
|
||||
# df_acheteurs / df_titulaires sont conservés en mémoire pour alimenter
|
||||
# la recherche sur la page d'accueil (autocomplétion, filtrage par sous-chaîne
|
||||
# à chaque frappe). Les colonnes reproduisent la sortie historique de
|
||||
# get_org_data(df, org_type).
|
||||
def _build_org_frame(org_type: str) -> pl.DataFrame:
|
||||
org_cols = [
|
||||
c
|
||||
for c in schema.names()
|
||||
if c.startswith(f"{org_type}_")
|
||||
and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
|
||||
]
|
||||
select_list = ", ".join(org_cols)
|
||||
group_list = ", ".join(org_cols)
|
||||
sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
|
||||
return get_cursor().execute(sql).pl()
|
||||
|
||||
|
||||
DF_ACHETEURS = _build_org_frame("acheteur")
|
||||
DF_TITULAIRES = _build_org_frame("titulaire")
|
||||
|
||||
COLUMNS = schema.names()
|
||||
|
||||
DEPARTEMENTS = get_departements()
|
||||
DEPARTEMENTS_GEOJSON = get_departements_geojson()
|
||||
DOMAIN_NAME = (
|
||||
"test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info"
|
||||
)
|
||||
META_CONTENT = {
|
||||
"image_url": f"https://{DOMAIN_NAME}/assets/decp.info.png",
|
||||
"title": "decp.info - exploration des marchés publics français",
|
||||
"description": (
|
||||
"Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. "
|
||||
"Pour une commande publique accessible à toutes et tous."
|
||||
),
|
||||
}
|
||||
DATA_SCHEMA = get_data_schema()
|
||||
Reference in New Issue
Block a user