Refactorisation des fonctions utils
This commit is contained in:
+2
-7
@@ -13,13 +13,8 @@ from dash import dash_table, dcc, html
|
|||||||
from dash_extensions.javascript import Namespace
|
from dash_extensions.javascript import Namespace
|
||||||
|
|
||||||
from src.db import schema
|
from src.db import schema
|
||||||
from src.old_utils import (
|
from src.utils.data import DATA_SCHEMA, DEPARTEMENTS_GEOJSON
|
||||||
DATA_SCHEMA,
|
from src.utils.table import add_links, format_number, setup_table_columns
|
||||||
DEPARTEMENTS_GEOJSON,
|
|
||||||
add_links,
|
|
||||||
format_number,
|
|
||||||
setup_table_columns,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_yearly_statistics(statistics, today_str) -> html.Div:
|
def get_yearly_statistics(statistics, today_str) -> html.Div:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import os
|
|||||||
from dash import dcc, html, register_page
|
from dash import dcc, html, register_page
|
||||||
|
|
||||||
from src.figures import get_sources_tables
|
from src.figures import get_sources_tables
|
||||||
from src.old_utils import META_CONTENT
|
from src.utils.seo import META_CONTENT
|
||||||
|
|
||||||
NAME = "À propos"
|
NAME = "À propos"
|
||||||
|
|
||||||
|
|||||||
@@ -24,16 +24,14 @@ from src.figures import (
|
|||||||
make_column_picker,
|
make_column_picker,
|
||||||
point_on_map,
|
point_on_map,
|
||||||
)
|
)
|
||||||
from src.old_utils import (
|
from src.utils.data import DF_ACHETEURS, get_annuaire_data, get_departement_region
|
||||||
|
from src.utils.frontend import get_button_properties
|
||||||
|
from src.utils.seo import META_CONTENT
|
||||||
|
from src.utils.table import (
|
||||||
COLUMNS,
|
COLUMNS,
|
||||||
DF_ACHETEURS,
|
|
||||||
META_CONTENT,
|
|
||||||
filter_table_data,
|
filter_table_data,
|
||||||
format_number,
|
format_number,
|
||||||
get_annuaire_data,
|
|
||||||
get_button_properties,
|
|
||||||
get_default_hidden_columns,
|
get_default_hidden_columns,
|
||||||
get_departement_region,
|
|
||||||
prepare_table_data,
|
prepare_table_data,
|
||||||
sort_table_data,
|
sort_table_data,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from dash import Input, Output, callback, dcc, html, register_page
|
from dash import Input, Output, callback, dcc, html, register_page
|
||||||
|
|
||||||
from src.db import get_cursor
|
from src.db import get_cursor
|
||||||
from src.old_utils import DEPARTEMENTS
|
from src.utils.data import DEPARTEMENTS
|
||||||
|
|
||||||
NAME = "Département"
|
NAME = "Département"
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from dash import dcc, html, register_page
|
from dash import dcc, html, register_page
|
||||||
|
|
||||||
from src.old_utils import DEPARTEMENTS
|
from src.utils.data import DEPARTEMENTS
|
||||||
|
|
||||||
NAME = "Départements"
|
NAME = "Départements"
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import polars as pl
|
|||||||
from dash import Input, Output, callback, dcc, html, register_page
|
from dash import Input, Output, callback, dcc, html, register_page
|
||||||
|
|
||||||
from src.db import get_cursor
|
from src.db import get_cursor
|
||||||
from src.old_utils import DF_ACHETEURS, DF_TITULAIRES
|
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
|
||||||
|
|
||||||
NAME = "Liste des marchés publics"
|
NAME = "Liste des marchés publics"
|
||||||
|
|
||||||
|
|||||||
+3
-7
@@ -6,13 +6,9 @@ from dash import Input, Output, callback, dcc, html, register_page
|
|||||||
from polars import selectors as cs
|
from polars import selectors as cs
|
||||||
|
|
||||||
from src.db import query_marches
|
from src.db import query_marches
|
||||||
from src.old_utils import (
|
from src.utils.data import DATA_SCHEMA
|
||||||
DATA_SCHEMA,
|
from src.utils.seo import META_CONTENT, make_org_jsonld
|
||||||
META_CONTENT,
|
from src.utils.table import format_values, unformat_montant
|
||||||
format_values,
|
|
||||||
make_org_jsonld,
|
|
||||||
unformat_montant,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_title(uid: str = None) -> str:
|
def get_title(uid: str = None) -> str:
|
||||||
|
|||||||
@@ -30,18 +30,16 @@ from src.figures import (
|
|||||||
make_column_picker,
|
make_column_picker,
|
||||||
make_donut,
|
make_donut,
|
||||||
)
|
)
|
||||||
from src.old_utils import (
|
from src.utils import logger
|
||||||
COLUMNS,
|
from src.utils.data import (
|
||||||
DEPARTEMENTS,
|
DEPARTEMENTS,
|
||||||
DF_ACHETEURS,
|
DF_ACHETEURS,
|
||||||
DF_TITULAIRES,
|
DF_TITULAIRES,
|
||||||
META_CONTENT,
|
|
||||||
get_default_hidden_columns,
|
|
||||||
get_enum_values_as_dict,
|
|
||||||
logger,
|
|
||||||
prepare_dashboard_data,
|
prepare_dashboard_data,
|
||||||
prepare_table_data,
|
|
||||||
)
|
)
|
||||||
|
from src.utils.frontend import get_enum_values_as_dict
|
||||||
|
from src.utils.seo import META_CONTENT
|
||||||
|
from src.utils.table import COLUMNS, get_default_hidden_columns, prepare_table_data
|
||||||
|
|
||||||
NAME = "Observatoire"
|
NAME = "Observatoire"
|
||||||
|
|
||||||
|
|||||||
@@ -2,13 +2,10 @@ import dash_bootstrap_components as dbc
|
|||||||
from dash import Input, Output, State, callback, dcc, html, register_page
|
from dash import Input, Output, State, callback, dcc, html, register_page
|
||||||
|
|
||||||
from src.figures import DataTable
|
from src.figures import DataTable
|
||||||
from src.old_utils import (
|
from src.utils.data import DF_ACHETEURS, DF_TITULAIRES
|
||||||
DF_ACHETEURS,
|
from src.utils.search import search_org
|
||||||
DF_TITULAIRES,
|
from src.utils.seo import META_CONTENT
|
||||||
META_CONTENT,
|
from src.utils.table import setup_table_columns
|
||||||
search_org,
|
|
||||||
setup_table_columns,
|
|
||||||
)
|
|
||||||
|
|
||||||
NAME = "Recherche"
|
NAME = "Recherche"
|
||||||
|
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ from dash import (
|
|||||||
|
|
||||||
from src.db import query_marches, schema
|
from src.db import query_marches, schema
|
||||||
from src.figures import DataTable, make_column_picker
|
from src.figures import DataTable, make_column_picker
|
||||||
from src.old_utils import (
|
from src.utils import logger
|
||||||
|
from src.utils.seo import META_CONTENT
|
||||||
|
from src.utils.table import (
|
||||||
COLUMNS,
|
COLUMNS,
|
||||||
META_CONTENT,
|
|
||||||
filter_table_data,
|
filter_table_data,
|
||||||
get_default_hidden_columns,
|
get_default_hidden_columns,
|
||||||
invert_columns,
|
invert_columns,
|
||||||
logger,
|
|
||||||
prepare_table_data,
|
prepare_table_data,
|
||||||
sort_table_data,
|
sort_table_data,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -23,16 +23,14 @@ from src.figures import (
|
|||||||
make_column_picker,
|
make_column_picker,
|
||||||
point_on_map,
|
point_on_map,
|
||||||
)
|
)
|
||||||
from src.old_utils import (
|
from src.utils.data import DF_TITULAIRES, get_annuaire_data, get_departement_region
|
||||||
|
from src.utils.frontend import get_button_properties
|
||||||
|
from src.utils.seo import META_CONTENT
|
||||||
|
from src.utils.table import (
|
||||||
COLUMNS,
|
COLUMNS,
|
||||||
DF_TITULAIRES,
|
|
||||||
META_CONTENT,
|
|
||||||
filter_table_data,
|
filter_table_data,
|
||||||
format_number,
|
format_number,
|
||||||
get_annuaire_data,
|
|
||||||
get_button_properties,
|
|
||||||
get_default_hidden_columns,
|
get_default_hidden_columns,
|
||||||
get_departement_region,
|
|
||||||
prepare_table_data,
|
prepare_table_data,
|
||||||
sort_table_data,
|
sort_table_data,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,214 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from collections import OrderedDict
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
import polars as pl
|
||||||
|
from httpx import HTTPError, get
|
||||||
|
|
||||||
|
from src.db import get_cursor, schema
|
||||||
|
from src.utils import logger
|
||||||
|
|
||||||
|
logging.getLogger("httpx").setLevel("WARNING")
|
||||||
|
|
||||||
|
|
||||||
|
def get_annuaire_data(siret: str) -> dict:
|
||||||
|
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
|
||||||
|
try:
|
||||||
|
response = get(url).raise_for_status()
|
||||||
|
response = response.json()["results"][0]
|
||||||
|
except (HTTPError, IndexError):
|
||||||
|
response = None
|
||||||
|
logger.warning("Could not fetch data from recherche-entreprises.api.")
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def get_statistics() -> dict:
|
||||||
|
return (
|
||||||
|
get(
|
||||||
|
"https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df",
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
.raise_for_status()
|
||||||
|
.json()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_departements() -> dict:
|
||||||
|
with open("data/departements.json", "rb") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def get_departements_geojson() -> dict:
|
||||||
|
with open("./data/departements-1000m.geojson") as f:
|
||||||
|
geojson = json.load(f)
|
||||||
|
|
||||||
|
# Ajout de feature.id
|
||||||
|
for f in geojson["features"]:
|
||||||
|
f["id"] = f["properties"]["code"]
|
||||||
|
|
||||||
|
return geojson
|
||||||
|
|
||||||
|
|
||||||
|
def get_departement_region(code_postal):
|
||||||
|
if code_postal > "97000":
|
||||||
|
code_departement = code_postal[:3]
|
||||||
|
else:
|
||||||
|
code_departement = code_postal[:2]
|
||||||
|
nom_departement = DEPARTEMENTS[code_departement]["departement"]
|
||||||
|
nom_region = DEPARTEMENTS[code_departement]["region"]
|
||||||
|
return code_departement, nom_departement, nom_region
|
||||||
|
|
||||||
|
|
||||||
|
def get_data_schema() -> dict:
|
||||||
|
# Récupération du schéma des données tabulaires
|
||||||
|
path = os.getenv("DATA_SCHEMA_PATH")
|
||||||
|
if path.startswith("http"):
|
||||||
|
original_schema: dict = get(
|
||||||
|
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
|
||||||
|
).json()
|
||||||
|
elif os.path.exists(path):
|
||||||
|
with open(path) as f:
|
||||||
|
original_schema: dict = json.load(f)
|
||||||
|
else:
|
||||||
|
raise Exception(f"Chemin vers le schéma invalide: {path}")
|
||||||
|
|
||||||
|
new_schema = OrderedDict()
|
||||||
|
|
||||||
|
for col in original_schema["fields"]:
|
||||||
|
new_schema[col["name"]] = col
|
||||||
|
|
||||||
|
return new_schema
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_dashboard_data(
|
||||||
|
lff: pl.LazyFrame,
|
||||||
|
dashboard_year=None,
|
||||||
|
dashboard_acheteur_id=None,
|
||||||
|
dashboard_acheteur_categorie=None,
|
||||||
|
dashboard_acheteur_departement_code=None,
|
||||||
|
dashboard_titulaire_id=None,
|
||||||
|
dashboard_titulaire_categorie=None,
|
||||||
|
dashboard_titulaire_departement_code=None,
|
||||||
|
dashboard_marche_type=None,
|
||||||
|
dashboard_marche_objet=None,
|
||||||
|
dashboard_marche_code_cpv=None,
|
||||||
|
dashboard_marche_considerations_sociales=None,
|
||||||
|
dashboard_marche_considerations_environnementales=None,
|
||||||
|
dashboard_marche_techniques=None,
|
||||||
|
dashboard_marche_innovant=None,
|
||||||
|
dashboard_marche_sous_traitance_declaree=None,
|
||||||
|
dashboard_montant_min=None,
|
||||||
|
dashboard_montant_max=None,
|
||||||
|
) -> pl.LazyFrame:
|
||||||
|
if dashboard_year:
|
||||||
|
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
|
||||||
|
else:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_acheteur_id:
|
||||||
|
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
|
||||||
|
else:
|
||||||
|
if dashboard_acheteur_categorie:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
|
||||||
|
)
|
||||||
|
if dashboard_acheteur_departement_code:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("acheteur_departement_code").is_in(
|
||||||
|
dashboard_acheteur_departement_code
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_titulaire_id:
|
||||||
|
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
|
||||||
|
else:
|
||||||
|
if dashboard_titulaire_categorie:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
|
||||||
|
)
|
||||||
|
if dashboard_titulaire_departement_code:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("titulaire_departement_code").is_in(
|
||||||
|
dashboard_titulaire_departement_code
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_marche_type:
|
||||||
|
lff = lff.filter(pl.col("type") == dashboard_marche_type)
|
||||||
|
|
||||||
|
if dashboard_marche_objet:
|
||||||
|
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
|
||||||
|
|
||||||
|
if dashboard_marche_code_cpv:
|
||||||
|
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
|
||||||
|
|
||||||
|
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
|
||||||
|
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
|
||||||
|
|
||||||
|
if (
|
||||||
|
dashboard_marche_sous_traitance_declaree
|
||||||
|
and dashboard_marche_sous_traitance_declaree != "all"
|
||||||
|
):
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_marche_techniques:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("techniques")
|
||||||
|
.str.split(", ")
|
||||||
|
.list.set_intersection(dashboard_marche_techniques)
|
||||||
|
.list.len()
|
||||||
|
> 0
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_marche_considerations_sociales:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("considerationsSociales")
|
||||||
|
.str.split(", ")
|
||||||
|
.list.set_intersection(dashboard_marche_considerations_sociales)
|
||||||
|
.list.len()
|
||||||
|
> 0
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_marche_considerations_environnementales:
|
||||||
|
lff = lff.filter(
|
||||||
|
pl.col("considerationsEnvironnementales")
|
||||||
|
.str.split(", ")
|
||||||
|
.list.set_intersection(dashboard_marche_considerations_environnementales)
|
||||||
|
.list.len()
|
||||||
|
> 0
|
||||||
|
)
|
||||||
|
|
||||||
|
if dashboard_montant_min is not None:
|
||||||
|
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
|
||||||
|
|
||||||
|
if dashboard_montant_max is not None:
|
||||||
|
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
|
||||||
|
|
||||||
|
return lff
|
||||||
|
|
||||||
|
|
||||||
|
def build_org_frame(org_type: str) -> pl.DataFrame:
|
||||||
|
org_cols = [
|
||||||
|
c
|
||||||
|
for c in schema.names()
|
||||||
|
if c.startswith(f"{org_type}_")
|
||||||
|
and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
|
||||||
|
]
|
||||||
|
select_list = ", ".join(org_cols)
|
||||||
|
group_list = ", ".join(org_cols)
|
||||||
|
sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
|
||||||
|
return get_cursor().execute(sql).pl()
|
||||||
|
|
||||||
|
|
||||||
|
DF_ACHETEURS = build_org_frame("acheteur")
|
||||||
|
DF_TITULAIRES = build_org_frame("titulaire")
|
||||||
|
DEPARTEMENTS = get_departements()
|
||||||
|
DEPARTEMENTS_GEOJSON = get_departements_geojson()
|
||||||
|
DATA_SCHEMA = get_data_schema()
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
import polars as pl
|
||||||
|
from unidecode import unidecode
|
||||||
|
|
||||||
|
from src.utils.table import add_links
|
||||||
|
from src.utils.tracking import track_search
|
||||||
|
|
||||||
|
|
||||||
|
def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
|
||||||
|
"""
|
||||||
|
Search in either 'acheteur' or 'titulaire' DataFrame.
|
||||||
|
|
||||||
|
:param dff: Polars DataFrame with acheteur or titulaire columns
|
||||||
|
:param query: User search string
|
||||||
|
:param org_type: 'acheteur' or 'titulaire'
|
||||||
|
:return: Filtered DataFrame with 'matches' column
|
||||||
|
"""
|
||||||
|
if not query.strip():
|
||||||
|
return dff.select(pl.lit(False).alias("matches"))
|
||||||
|
|
||||||
|
# Enregistrement des recherche dans Matomo
|
||||||
|
track_search(query, "home_page_search")
|
||||||
|
|
||||||
|
# Normalize query
|
||||||
|
normalized_query = unidecode(query.strip()).upper()
|
||||||
|
tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()]
|
||||||
|
|
||||||
|
# Define columns based on entity type
|
||||||
|
cols = [
|
||||||
|
f"{org_type}_id",
|
||||||
|
f"{org_type}_nom",
|
||||||
|
f"{org_type}_departement_nom",
|
||||||
|
f"{org_type}_departement_code",
|
||||||
|
f"{org_type}_commune_nom",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Concatenate all fields into one string per row
|
||||||
|
org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace(
|
||||||
|
"-", " "
|
||||||
|
)
|
||||||
|
|
||||||
|
# For each token, create a boolean column: True if token is found
|
||||||
|
token_matches = []
|
||||||
|
for token in tokens:
|
||||||
|
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
||||||
|
token_matches.append(token_match)
|
||||||
|
|
||||||
|
# Count how many tokens match per row
|
||||||
|
match_score = pl.sum_horizontal(token_matches).alias("match_score")
|
||||||
|
|
||||||
|
# For each token, create a boolean column: True if token is found
|
||||||
|
token_matches = []
|
||||||
|
for token in tokens:
|
||||||
|
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
||||||
|
token_matches.append(token_match)
|
||||||
|
|
||||||
|
# Sélection des colonnes
|
||||||
|
if org_type == "acheteur":
|
||||||
|
dff = dff.select(cols + ["Marchés"])
|
||||||
|
if org_type == "titulaire":
|
||||||
|
dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"])
|
||||||
|
|
||||||
|
# Apply and filter
|
||||||
|
dff = (
|
||||||
|
dff.with_columns(token_matches + [match_score])
|
||||||
|
.filter(pl.col("match_score") == len(tokens))
|
||||||
|
.drop([f"token_{token}" for token in tokens])
|
||||||
|
)
|
||||||
|
|
||||||
|
# Format result
|
||||||
|
dff = add_links(dff)
|
||||||
|
dff = dff.with_columns(
|
||||||
|
pl.concat_str(
|
||||||
|
pl.col(f"{org_type}_departement_nom"),
|
||||||
|
pl.lit(" ("),
|
||||||
|
pl.col(f"{org_type}_departement_code"),
|
||||||
|
pl.lit(")"),
|
||||||
|
).alias("Département")
|
||||||
|
)
|
||||||
|
|
||||||
|
dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés")
|
||||||
|
dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum()
|
||||||
|
dff = dff.sort("Marchés", descending=True)
|
||||||
|
|
||||||
|
return dff
|
||||||
@@ -1,31 +1,15 @@
|
|||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from collections import OrderedDict
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from time import localtime
|
|
||||||
|
|
||||||
import polars as pl
|
import polars as pl
|
||||||
import polars.selectors as cs
|
|
||||||
from dash import no_update
|
from dash import no_update
|
||||||
from httpx import HTTPError, get, post
|
from polars import selectors as cs
|
||||||
from unidecode import unidecode
|
|
||||||
|
|
||||||
from src.db import conn as duckdb_conn # noqa: F401 (exposed for convenience)
|
from src.db import query_marches, schema
|
||||||
from src.db import get_cursor, query_marches, schema # noqa: F401
|
from src.utils import logger
|
||||||
|
from src.utils.data import DATA_SCHEMA
|
||||||
logging.basicConfig(
|
from src.utils.frontend import get_button_properties
|
||||||
format="%(asctime)s %(levelname)-8s %(message)s",
|
from src.utils.tracking import track_search
|
||||||
level=logging.INFO,
|
|
||||||
datefmt="%Y-%m-%d %H:%M:%S",
|
|
||||||
)
|
|
||||||
logger = logging.getLogger("decp.info")
|
|
||||||
DEVELOPMENT = os.getenv("DEVELOPMENT", "False").lower() == "true"
|
|
||||||
if DEVELOPMENT:
|
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
logging.getLogger("httpx").setLevel("WARNING")
|
|
||||||
|
|
||||||
|
|
||||||
def split_filter_part(filter_part):
|
def split_filter_part(filter_part):
|
||||||
@@ -222,55 +206,6 @@ def format_values(dff: pl.DataFrame) -> pl.DataFrame:
|
|||||||
return dff
|
return dff
|
||||||
|
|
||||||
|
|
||||||
def get_annuaire_data(siret: str) -> dict:
|
|
||||||
url = f"https://recherche-entreprises.api.gouv.fr/search?q={siret}"
|
|
||||||
try:
|
|
||||||
response = get(url).raise_for_status()
|
|
||||||
response = response.json()["results"][0]
|
|
||||||
except (HTTPError, IndexError):
|
|
||||||
response = None
|
|
||||||
logger.warning("Could not fetch data from recherche-entreprises.api.")
|
|
||||||
return response
|
|
||||||
|
|
||||||
|
|
||||||
def get_statistics() -> dict:
|
|
||||||
return (
|
|
||||||
get(
|
|
||||||
"https://www.data.gouv.fr/api/1/datasets/r/0ccf4a75-f3aa-4b46-8b6a-18aeb63e36df",
|
|
||||||
follow_redirects=True,
|
|
||||||
)
|
|
||||||
.raise_for_status()
|
|
||||||
.json()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_departements() -> dict:
|
|
||||||
with open("data/departements.json", "rb") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
def get_departements_geojson() -> dict:
|
|
||||||
with open("./data/departements-1000m.geojson") as f:
|
|
||||||
geojson = json.load(f)
|
|
||||||
|
|
||||||
# Ajout de feature.id
|
|
||||||
for f in geojson["features"]:
|
|
||||||
f["id"] = f["properties"]["code"]
|
|
||||||
|
|
||||||
return geojson
|
|
||||||
|
|
||||||
|
|
||||||
def get_departement_region(code_postal):
|
|
||||||
if code_postal > "97000":
|
|
||||||
code_departement = code_postal[:3]
|
|
||||||
else:
|
|
||||||
code_departement = code_postal[:2]
|
|
||||||
nom_departement = DEPARTEMENTS[code_departement]["departement"]
|
|
||||||
nom_region = DEPARTEMENTS[code_departement]["region"]
|
|
||||||
return code_departement, nom_departement, nom_region
|
|
||||||
|
|
||||||
|
|
||||||
def filter_table_data(
|
def filter_table_data(
|
||||||
lff: pl.LazyFrame, filter_query: str, filter_source: str
|
lff: pl.LazyFrame, filter_query: str, filter_source: str
|
||||||
) -> pl.LazyFrame:
|
) -> pl.LazyFrame:
|
||||||
@@ -433,129 +368,6 @@ def get_default_hidden_columns(page):
|
|||||||
return hidden_columns
|
return hidden_columns
|
||||||
|
|
||||||
|
|
||||||
def get_data_schema() -> dict:
|
|
||||||
# Récupération du schéma des données tabulaires
|
|
||||||
path = os.getenv("DATA_SCHEMA_PATH")
|
|
||||||
if path.startswith("http"):
|
|
||||||
original_schema: dict = get(
|
|
||||||
os.getenv("DATA_SCHEMA_PATH"), follow_redirects=True
|
|
||||||
).json()
|
|
||||||
elif os.path.exists(path):
|
|
||||||
with open(path) as f:
|
|
||||||
original_schema: dict = json.load(f)
|
|
||||||
else:
|
|
||||||
raise Exception(f"Chemin vers le schéma invalide: {path}")
|
|
||||||
|
|
||||||
new_schema = OrderedDict()
|
|
||||||
|
|
||||||
for col in original_schema["fields"]:
|
|
||||||
new_schema[col["name"]] = col
|
|
||||||
|
|
||||||
return new_schema
|
|
||||||
|
|
||||||
|
|
||||||
def track_search(query, category):
|
|
||||||
if len(query) >= 4 and not DEVELOPMENT and os.getenv("MATOMO_DOMAIN"):
|
|
||||||
url = "https://decp.info"
|
|
||||||
params = {
|
|
||||||
"idsite": os.getenv("MATOMO_ID_SITE"),
|
|
||||||
"url": url,
|
|
||||||
"rec": "1",
|
|
||||||
"action_name": "search" if category == "home_page_search" else "filter",
|
|
||||||
"search_cat": category,
|
|
||||||
"rand": uuid.uuid4().hex,
|
|
||||||
"apiv": "1",
|
|
||||||
"h": localtime().tm_hour,
|
|
||||||
"m": localtime().tm_min,
|
|
||||||
"s": localtime().tm_sec,
|
|
||||||
"search": query,
|
|
||||||
"token_auth": os.getenv("MATOMO_TOKEN"),
|
|
||||||
}
|
|
||||||
post(
|
|
||||||
url=f"https://{os.getenv('MATOMO_DOMAIN')}/matomo.php",
|
|
||||||
params=params,
|
|
||||||
).raise_for_status()
|
|
||||||
|
|
||||||
|
|
||||||
def search_org(dff: pl.DataFrame, query: str, org_type: str) -> pl.DataFrame:
|
|
||||||
"""
|
|
||||||
Search in either 'acheteur' or 'titulaire' DataFrame.
|
|
||||||
|
|
||||||
:param dff: Polars DataFrame with acheteur or titulaire columns
|
|
||||||
:param query: User search string
|
|
||||||
:param org_type: 'acheteur' or 'titulaire'
|
|
||||||
:return: Filtered DataFrame with 'matches' column
|
|
||||||
"""
|
|
||||||
if not query.strip():
|
|
||||||
return dff.select(pl.lit(False).alias("matches"))
|
|
||||||
|
|
||||||
# Enregistrement des recherche dans Matomo
|
|
||||||
track_search(query, "home_page_search")
|
|
||||||
|
|
||||||
# Normalize query
|
|
||||||
normalized_query = unidecode(query.strip()).upper()
|
|
||||||
tokens = [" " + t.strip() for t in normalized_query.split() if t.strip()]
|
|
||||||
|
|
||||||
# Define columns based on entity type
|
|
||||||
cols = [
|
|
||||||
f"{org_type}_id",
|
|
||||||
f"{org_type}_nom",
|
|
||||||
f"{org_type}_departement_nom",
|
|
||||||
f"{org_type}_departement_code",
|
|
||||||
f"{org_type}_commune_nom",
|
|
||||||
]
|
|
||||||
|
|
||||||
# Concatenate all fields into one string per row
|
|
||||||
org_str = pl.concat_str(pl.lit(" "), pl.col(cols), separator=" ").str.replace(
|
|
||||||
"-", " "
|
|
||||||
)
|
|
||||||
|
|
||||||
# For each token, create a boolean column: True if token is found
|
|
||||||
token_matches = []
|
|
||||||
for token in tokens:
|
|
||||||
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
|
||||||
token_matches.append(token_match)
|
|
||||||
|
|
||||||
# Count how many tokens match per row
|
|
||||||
match_score = pl.sum_horizontal(token_matches).alias("match_score")
|
|
||||||
|
|
||||||
# For each token, create a boolean column: True if token is found
|
|
||||||
token_matches = []
|
|
||||||
for token in tokens:
|
|
||||||
token_match = org_str.str.contains(token).alias(f"token_{token}")
|
|
||||||
token_matches.append(token_match)
|
|
||||||
|
|
||||||
# Sélection des colonnes
|
|
||||||
if org_type == "acheteur":
|
|
||||||
dff = dff.select(cols + ["Marchés"])
|
|
||||||
if org_type == "titulaire":
|
|
||||||
dff = dff.select(cols + ["Marchés", "titulaire_typeIdentifiant"])
|
|
||||||
|
|
||||||
# Apply and filter
|
|
||||||
dff = (
|
|
||||||
dff.with_columns(token_matches + [match_score])
|
|
||||||
.filter(pl.col("match_score") == len(tokens))
|
|
||||||
.drop([f"token_{token}" for token in tokens])
|
|
||||||
)
|
|
||||||
|
|
||||||
# Format result
|
|
||||||
dff = add_links(dff)
|
|
||||||
dff = dff.with_columns(
|
|
||||||
pl.concat_str(
|
|
||||||
pl.col(f"{org_type}_departement_nom"),
|
|
||||||
pl.lit(" ("),
|
|
||||||
pl.col(f"{org_type}_departement_code"),
|
|
||||||
pl.lit(")"),
|
|
||||||
).alias("Département")
|
|
||||||
)
|
|
||||||
|
|
||||||
dff = dff.select(f"{org_type}_id", f"{org_type}_nom", "Département", "Marchés")
|
|
||||||
dff = dff.group_by(f"{org_type}_id", f"{org_type}_nom", "Département").sum()
|
|
||||||
dff = dff.sort("Marchés", descending=True)
|
|
||||||
|
|
||||||
return dff
|
|
||||||
|
|
||||||
|
|
||||||
def prepare_table_data(
|
def prepare_table_data(
|
||||||
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
|
data, data_timestamp, filter_query, page_current, page_size, sort_by, source_table
|
||||||
):
|
):
|
||||||
@@ -646,143 +458,6 @@ def prepare_table_data(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def prepare_dashboard_data(
|
|
||||||
lff: pl.LazyFrame,
|
|
||||||
dashboard_year=None,
|
|
||||||
dashboard_acheteur_id=None,
|
|
||||||
dashboard_acheteur_categorie=None,
|
|
||||||
dashboard_acheteur_departement_code=None,
|
|
||||||
dashboard_titulaire_id=None,
|
|
||||||
dashboard_titulaire_categorie=None,
|
|
||||||
dashboard_titulaire_departement_code=None,
|
|
||||||
dashboard_marche_type=None,
|
|
||||||
dashboard_marche_objet=None,
|
|
||||||
dashboard_marche_code_cpv=None,
|
|
||||||
dashboard_marche_considerations_sociales=None,
|
|
||||||
dashboard_marche_considerations_environnementales=None,
|
|
||||||
dashboard_marche_techniques=None,
|
|
||||||
dashboard_marche_innovant=None,
|
|
||||||
dashboard_marche_sous_traitance_declaree=None,
|
|
||||||
dashboard_montant_min=None,
|
|
||||||
dashboard_montant_max=None,
|
|
||||||
) -> pl.LazyFrame:
|
|
||||||
if dashboard_year:
|
|
||||||
lff = lff.filter(pl.col("dateNotification").dt.year() == int(dashboard_year))
|
|
||||||
else:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("dateNotification") > (datetime.now() - timedelta(days=365))
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_acheteur_id:
|
|
||||||
lff = lff.filter(pl.col("acheteur_id").str.contains(dashboard_acheteur_id))
|
|
||||||
else:
|
|
||||||
if dashboard_acheteur_categorie:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("acheteur_categorie") == dashboard_acheteur_categorie
|
|
||||||
)
|
|
||||||
if dashboard_acheteur_departement_code:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("acheteur_departement_code").is_in(
|
|
||||||
dashboard_acheteur_departement_code
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_titulaire_id:
|
|
||||||
lff = lff.filter(pl.col("titulaire_id").str.contains(dashboard_titulaire_id))
|
|
||||||
else:
|
|
||||||
if dashboard_titulaire_categorie:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("titulaire_categorie") == dashboard_titulaire_categorie
|
|
||||||
)
|
|
||||||
if dashboard_titulaire_departement_code:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("titulaire_departement_code").is_in(
|
|
||||||
dashboard_titulaire_departement_code
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_marche_type:
|
|
||||||
lff = lff.filter(pl.col("type") == dashboard_marche_type)
|
|
||||||
|
|
||||||
if dashboard_marche_objet:
|
|
||||||
lff = lff.filter(pl.col("objet").str.contains(f"(?i){dashboard_marche_objet}"))
|
|
||||||
|
|
||||||
if dashboard_marche_code_cpv:
|
|
||||||
lff = lff.filter(pl.col("codeCPV").str.starts_with(dashboard_marche_code_cpv))
|
|
||||||
|
|
||||||
if dashboard_marche_innovant and dashboard_marche_innovant != "all":
|
|
||||||
lff = lff.filter(pl.col("marcheInnovant") == dashboard_marche_innovant)
|
|
||||||
|
|
||||||
if (
|
|
||||||
dashboard_marche_sous_traitance_declaree
|
|
||||||
and dashboard_marche_sous_traitance_declaree != "all"
|
|
||||||
):
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("sousTraitanceDeclaree") == dashboard_marche_sous_traitance_declaree
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_marche_techniques:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("techniques")
|
|
||||||
.str.split(", ")
|
|
||||||
.list.set_intersection(dashboard_marche_techniques)
|
|
||||||
.list.len()
|
|
||||||
> 0
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_marche_considerations_sociales:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("considerationsSociales")
|
|
||||||
.str.split(", ")
|
|
||||||
.list.set_intersection(dashboard_marche_considerations_sociales)
|
|
||||||
.list.len()
|
|
||||||
> 0
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_marche_considerations_environnementales:
|
|
||||||
lff = lff.filter(
|
|
||||||
pl.col("considerationsEnvironnementales")
|
|
||||||
.str.split(", ")
|
|
||||||
.list.set_intersection(dashboard_marche_considerations_environnementales)
|
|
||||||
.list.len()
|
|
||||||
> 0
|
|
||||||
)
|
|
||||||
|
|
||||||
if dashboard_montant_min is not None:
|
|
||||||
lff = lff.filter(pl.col("montant") >= dashboard_montant_min)
|
|
||||||
|
|
||||||
if dashboard_montant_max is not None:
|
|
||||||
lff = lff.filter(pl.col("montant") <= dashboard_montant_max)
|
|
||||||
|
|
||||||
return lff
|
|
||||||
|
|
||||||
|
|
||||||
def get_button_properties(height):
|
|
||||||
if height > 65000:
|
|
||||||
download_disabled = True
|
|
||||||
download_text = "Téléchargement désactivé au-delà de 65 000 lignes"
|
|
||||||
download_title = " Ajoutez des filtres pour réduire le nombre de lignes, Excel ne supporte pas d'avoir plus de 65 000 URLs dans une même feuille de calcul."
|
|
||||||
elif height == 0:
|
|
||||||
download_disabled = True
|
|
||||||
download_text = "Pas de données à télécharger"
|
|
||||||
download_title = ""
|
|
||||||
else:
|
|
||||||
download_disabled = False
|
|
||||||
download_text = "Télécharger au format Excel"
|
|
||||||
download_title = "Télécharger les données telles qu'affichées au format Excel"
|
|
||||||
return download_disabled, download_text, download_title
|
|
||||||
|
|
||||||
|
|
||||||
def get_enum_values_as_dict(column_name):
|
|
||||||
try:
|
|
||||||
options = {}
|
|
||||||
for value in DATA_SCHEMA[column_name]["enum"]:
|
|
||||||
options[value] = value
|
|
||||||
return options
|
|
||||||
except KeyError:
|
|
||||||
return {"not_found": "not found"}
|
|
||||||
|
|
||||||
|
|
||||||
def invert_columns(columns):
|
def invert_columns(columns):
|
||||||
"""
|
"""
|
||||||
Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa.
|
Renvoie les colonnes du schéma non spécifiées en paramètre. Utile pour passer d'une colonnes masquées à une liste de colonnes affichées, et vice versa.
|
||||||
@@ -797,79 +472,4 @@ def invert_columns(columns):
|
|||||||
return inverted_columns
|
return inverted_columns
|
||||||
|
|
||||||
|
|
||||||
def make_org_jsonld(org_id, org_type, org_name=None, type_org_id="SIRET") -> dict:
|
|
||||||
org_types = {"acheteur": "GovernmentOrganization", "titulaire": "Organization"}
|
|
||||||
address = None
|
|
||||||
if type_org_id.lower() == "siret" and len(org_id) == 14:
|
|
||||||
annuaire_data = get_annuaire_data(org_id)
|
|
||||||
annuaire_address = annuaire_data["matching_etablissements"][0]
|
|
||||||
code_postal = annuaire_address["code_postal"]
|
|
||||||
commune = annuaire_address["libelle_commune"]
|
|
||||||
|
|
||||||
address = (
|
|
||||||
{
|
|
||||||
"@type": "PostalAddress",
|
|
||||||
"streetAddress": annuaire_address.get("adresse", "")
|
|
||||||
.replace(code_postal, "")
|
|
||||||
.replace(commune, "")
|
|
||||||
.strip(),
|
|
||||||
"addressLocality": commune,
|
|
||||||
"postalCode": code_postal,
|
|
||||||
"addressCountry": "FR",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
jsonld = {
|
|
||||||
"@type": org_types[org_type],
|
|
||||||
"name": org_name,
|
|
||||||
"url": f"https://decp.info/{org_type}s/{org_id}",
|
|
||||||
"sameAs": f"https://annuaire-entreprises.data.gouv.fr/etablissement/{org_id}",
|
|
||||||
"identifier": {
|
|
||||||
"@type": "PropertyValue",
|
|
||||||
"propertyID": type_org_id.lower(),
|
|
||||||
"value": org_id,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if address:
|
|
||||||
jsonld["address"] = address
|
|
||||||
|
|
||||||
return jsonld
|
|
||||||
|
|
||||||
|
|
||||||
# df_acheteurs / df_titulaires sont conservés en mémoire pour alimenter
|
|
||||||
# la recherche sur la page d'accueil (autocomplétion, filtrage par sous-chaîne
|
|
||||||
# à chaque frappe). Les colonnes reproduisent la sortie historique de
|
|
||||||
# get_org_data(df, org_type).
|
|
||||||
def _build_org_frame(org_type: str) -> pl.DataFrame:
|
|
||||||
org_cols = [
|
|
||||||
c
|
|
||||||
for c in schema.names()
|
|
||||||
if c.startswith(f"{org_type}_")
|
|
||||||
and c not in (f"{org_type}_latitude", f"{org_type}_longitude")
|
|
||||||
]
|
|
||||||
select_list = ", ".join(org_cols)
|
|
||||||
group_list = ", ".join(org_cols)
|
|
||||||
sql = f'SELECT {select_list}, COUNT(*) AS "Marchés" FROM decp GROUP BY {group_list}'
|
|
||||||
return get_cursor().execute(sql).pl()
|
|
||||||
|
|
||||||
|
|
||||||
DF_ACHETEURS = _build_org_frame("acheteur")
|
|
||||||
DF_TITULAIRES = _build_org_frame("titulaire")
|
|
||||||
|
|
||||||
COLUMNS = schema.names()
|
COLUMNS = schema.names()
|
||||||
|
|
||||||
DEPARTEMENTS = get_departements()
|
|
||||||
DEPARTEMENTS_GEOJSON = get_departements_geojson()
|
|
||||||
DOMAIN_NAME = (
|
|
||||||
"test.decp.info" if os.getenv("DEVELOPMENT").lower() == "true" else "decp.info"
|
|
||||||
)
|
|
||||||
META_CONTENT = {
|
|
||||||
"image_url": f"https://{DOMAIN_NAME}/assets/decp.info.png",
|
|
||||||
"title": "decp.info - exploration des marchés publics français",
|
|
||||||
"description": (
|
|
||||||
"Explorez et analysez les données des marchés publics français avec cet outil libre et gratuit. "
|
|
||||||
"Pour une commande publique accessible à toutes et tous."
|
|
||||||
),
|
|
||||||
}
|
|
||||||
DATA_SCHEMA = get_data_schema()
|
|
||||||
Reference in New Issue
Block a user