@@ -0,0 +1,28 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger("decp.info")
|
||||||
|
|
||||||
|
|
||||||
|
def should_rebuild(db_path: Path, parquet_path: Path) -> bool:
|
||||||
|
"""Decide whether to rebuild the DuckDB database from the source Parquet.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- Rebuild if the DuckDB file does not exist.
|
||||||
|
- Otherwise, rebuild only if the source Parquet is newer than the DB,
|
||||||
|
EXCEPT in development mode without REBUILD_DUCKDB=true (dev keeps
|
||||||
|
a stable DB across reloads unless explicitly opted in).
|
||||||
|
"""
|
||||||
|
db_path = Path(db_path)
|
||||||
|
parquet_path = Path(parquet_path)
|
||||||
|
|
||||||
|
if not db_path.exists():
|
||||||
|
return True
|
||||||
|
|
||||||
|
dev = os.getenv("DEVELOPMENT", "False").lower() == "true"
|
||||||
|
force = os.getenv("REBUILD_DUCKDB", "False").lower() == "true"
|
||||||
|
if dev and not force:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return parquet_path.stat().st_mtime > db_path.stat().st_mtime
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def parquet_and_db(tmp_path, monkeypatch):
|
||||||
|
parquet = tmp_path / "source.parquet"
|
||||||
|
db = tmp_path / "decp.duckdb"
|
||||||
|
parquet.write_bytes(b"fake parquet content")
|
||||||
|
monkeypatch.delenv("REBUILD_DUCKDB", raising=False)
|
||||||
|
monkeypatch.delenv("DEVELOPMENT", raising=False)
|
||||||
|
return parquet, db
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_rebuild_when_db_missing(parquet_and_db, monkeypatch):
|
||||||
|
from src.db import should_rebuild
|
||||||
|
|
||||||
|
parquet, db = parquet_and_db
|
||||||
|
monkeypatch.setenv("DEVELOPMENT", "true")
|
||||||
|
assert should_rebuild(db, parquet) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_rebuild_prod_when_parquet_newer(parquet_and_db, monkeypatch):
|
||||||
|
from src.db import should_rebuild
|
||||||
|
|
||||||
|
parquet, db = parquet_and_db
|
||||||
|
db.write_bytes(b"x")
|
||||||
|
time.sleep(0.01)
|
||||||
|
parquet.touch()
|
||||||
|
monkeypatch.setenv("DEVELOPMENT", "false")
|
||||||
|
assert should_rebuild(db, parquet) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_not_rebuild_prod_when_parquet_older(parquet_and_db, monkeypatch):
|
||||||
|
from src.db import should_rebuild
|
||||||
|
|
||||||
|
parquet, db = parquet_and_db
|
||||||
|
parquet.touch()
|
||||||
|
time.sleep(0.01)
|
||||||
|
db.write_bytes(b"x")
|
||||||
|
monkeypatch.setenv("DEVELOPMENT", "false")
|
||||||
|
assert should_rebuild(db, parquet) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_not_rebuild_dev_even_when_parquet_newer(parquet_and_db, monkeypatch):
|
||||||
|
from src.db import should_rebuild
|
||||||
|
|
||||||
|
parquet, db = parquet_and_db
|
||||||
|
db.write_bytes(b"x")
|
||||||
|
time.sleep(0.01)
|
||||||
|
parquet.touch()
|
||||||
|
monkeypatch.setenv("DEVELOPMENT", "true")
|
||||||
|
monkeypatch.delenv("REBUILD_DUCKDB", raising=False)
|
||||||
|
assert should_rebuild(db, parquet) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_rebuild_dev_when_rebuild_forced(parquet_and_db, monkeypatch):
|
||||||
|
from src.db import should_rebuild
|
||||||
|
|
||||||
|
parquet, db = parquet_and_db
|
||||||
|
db.write_bytes(b"x")
|
||||||
|
time.sleep(0.01)
|
||||||
|
parquet.touch()
|
||||||
|
monkeypatch.setenv("DEVELOPMENT", "true")
|
||||||
|
monkeypatch.setenv("REBUILD_DUCKDB", "true")
|
||||||
|
assert should_rebuild(db, parquet) is True
|
||||||
Reference in New Issue
Block a user