Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion airflow-core/src/airflow/utils/db_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any

from sqlalchemy import and_, column, func, inspect, select, table, text
from sqlalchemy import and_, column, func, inspect, literal, select, table, text
from sqlalchemy.exc import OperationalError, ProgrammingError
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.orm import aliased
Expand Down Expand Up @@ -78,6 +78,12 @@ class _TableConfig:
:param keep_last_group_by: if keeping the last record, can keep the last record for each group
:param dependent_tables: list of tables which have FK relationship with this table
:param extra_filters: SQLAlchemy expressions ANDed with the recency filter; referenced columns must be in ``extra_columns``.
:param skip_if_referenced: list of ``(referencing_table, fk_column)`` pairs whose FK points at this
table's ``referenced_pk_column``. A row that is still referenced by any of these is excluded from
deletion. This avoids issuing deletes that would violate an ``ON DELETE RESTRICT`` foreign key
(e.g. ``task_instance.dag_version_id``) — such deletes fail and, on MySQL, can leave the cleanup
command blocked on metadata locks. ``referenced_pk_column`` must be listed in ``extra_columns``.
:param referenced_pk_column: the primary-key column of this table that ``skip_if_referenced`` FKs point at.
"""

table_name: str
Expand All @@ -92,6 +98,8 @@ class _TableConfig:
# Relying on automation here would increase complexity and reduce maintainability.
dependent_tables: list[str] | None = None
extra_filters: list[Any] | None = None
skip_if_referenced: list[tuple[str, str]] | None = None
referenced_pk_column: str = "id"

def __post_init__(self):
self.recency_column = column(self.recency_column_name)
Comment thread
ephraimbuddy marked this conversation as resolved.
Resolved
Expand All @@ -109,6 +117,16 @@ def __post_init__(self):
self.recency_column,
)

# skip_if_referenced filters on referenced_pk_column, which must be a column of orm_model
# (added via extra_columns). Fail fast with a clear message instead of a cryptic KeyError
# raised later when _build_query evaluates base_table.c[referenced_pk_column].
if self.skip_if_referenced and self.referenced_pk_column not in self.orm_model.c.keys():
raise ValueError(
f"_TableConfig for table {self.table_name!r} sets skip_if_referenced but its "
f"referenced_pk_column {self.referenced_pk_column!r} is not one of its columns; "
f"add {self.referenced_pk_column!r} to extra_columns."
)

def __lt__(self, other):
return self.table_name < other.table_name

Expand Down Expand Up @@ -174,10 +192,16 @@ def readable_config(self):
_TableConfig(
table_name="dag_version",
recency_column_name="created_at",
extra_columns=["id"],
dependent_tables=["task_instance", "dag_run"],
dag_id_column_name="dag_id",
keep_last=True,
keep_last_group_by=["dag_id"],
# task_instance.dag_version_id is ON DELETE RESTRICT, so a version still referenced by any
# task instance cannot be deleted. Skip those rows instead of issuing a delete that would
# fail the FK (and hang on MySQL). They become eligible once their task instances age out
# and are cleaned. dag_run.created_dag_version_id is ON DELETE SET NULL, so it does not block.
skip_if_referenced=[("task_instance", "dag_version_id")],
),
_TableConfig(table_name="deadline", recency_column_name="deadline_time", dag_id_column_name="dag_id"),
_TableConfig(table_name="revoked_token", recency_column_name="exp"),
Expand Down Expand Up @@ -357,6 +381,8 @@ def _build_query(
dag_ids: list[str] | None = None,
exclude_dag_ids: list[str] | None = None,
extra_filters: list[Any] | None = None,
skip_if_referenced: list[tuple[str, str]] | None = None,
referenced_pk_column: str = "id",
**kwargs,
) -> Select:
base_table_alias = "base"
Expand All @@ -368,6 +394,22 @@ def _build_query(
if extra_filters:
conditions.extend(extra_filters)

if skip_if_referenced:
# Exclude rows still referenced by a RESTRICT foreign key; deleting them would fail the
# constraint (and on MySQL leave the command blocked on metadata locks). correlate() is
# explicit on purpose: this is a NOT EXISTS guard whose silent failure would delete
# still-referenced rows, so we don't rely on implicit correlation of the base table.
base_table_pk_col = base_table.c[referenced_pk_column]
for referencing_table, fk_column in skip_if_referenced:
referencing = table(referencing_table, column(fk_column))
conditions.append(
~select(literal(1))
.select_from(referencing)
.where(referencing.c[fk_column] == base_table_pk_col)
.correlate(base_table)
.exists()
)

if (dag_ids or exclude_dag_ids) and dag_id_column is not None:
base_table_dag_id_col = base_table.c[dag_id_column.name]

Expand Down Expand Up @@ -414,6 +456,8 @@ def _cleanup_table(
session: Session,
batch_size: int | None = None,
extra_filters: list[Any] | None = None,
skip_if_referenced: list[tuple[str, str]] | None = None,
referenced_pk_column: str = "id",
**kwargs,
) -> None:
print()
Expand All @@ -430,6 +474,8 @@ def _cleanup_table(
keep_last_group_by=keep_last_group_by,
clean_before_timestamp=clean_before_timestamp,
extra_filters=extra_filters,
skip_if_referenced=skip_if_referenced,
referenced_pk_column=referenced_pk_column,
session=session,
)
logger.debug("old rows query:\n%s", query.selectable.compile())
Expand Down
83 changes: 83 additions & 0 deletions airflow-core/tests/unit/utils/test_db_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
_confirm_drop_archives,
_dump_table_to_file,
_get_archived_table_names,
_TableConfig,
config_dict,
drop_archived_tables,
export_archived_records,
Expand Down Expand Up @@ -475,6 +476,88 @@ def test__skip_archive(self, skip_archive, expected_archives):
assert session.scalar(select(func.count()).select_from(model)) == 5
assert len(_get_archived_table_names(["dag_run"], session)) == expected_archives

def test_dag_version_cleanup_skips_versions_pinned_by_task_instance(self):
"""db clean must skip dag_version rows still referenced by a task instance.

``task_instance.dag_version_id`` is ``ON DELETE RESTRICT``, so deleting a referenced
``dag_version`` fails the foreign key. Cleanup must skip those rows while still pruning
genuinely orphaned older versions.
"""
base_date = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("UTC"))
bundle_name = f"testing-{uuid4()}"
dag_id = f"test_dag_{uuid4()}"

with create_session() as session:
session.add(DagBundleModel(name=bundle_name))
session.flush()
session.add(DagModel(dag_id=dag_id, bundle_name=bundle_name))
session.flush()

pinned_old = DagVersion(
dag_id=dag_id,
version_number=1,
bundle_name=bundle_name,
created_at=base_date,
last_updated=base_date,
)
orphan_old = DagVersion(
dag_id=dag_id,
version_number=2,
bundle_name=bundle_name,
created_at=base_date.add(minutes=1),
last_updated=base_date.add(minutes=1),
)
latest = DagVersion(
dag_id=dag_id,
version_number=3,
bundle_name=bundle_name,
created_at=base_date.add(minutes=2),
last_updated=base_date.add(minutes=2),
)
session.add_all([pinned_old, orphan_old, latest])
session.flush()

dag = DAG(dag_id=dag_id)
dag_run = DagRun(dag_id, run_id="run-1", run_type=DagRunType.MANUAL, start_date=base_date)
ti = create_task_instance(
PythonOperator(task_id="dummy-task", python_callable=print),
run_id=dag_run.run_id,
dag_version_id=pinned_old.id,
)
ti.dag_id = dag.dag_id
ti.start_date = base_date
session.add_all([dag_run, ti])
session.commit()

pinned_id, orphan_id, latest_id = pinned_old.id, orphan_old.id, latest.id

# Previously this raised an IntegrityError on the FK; it must now succeed.
_cleanup_table(
**config_dict["dag_version"].__dict__,
clean_before_timestamp=base_date.add(days=10),
dry_run=False,
session=session,
table_names=["dag_version"],
skip_archive=True,
)

remaining = set(session.scalars(select(DagVersion.id).where(DagVersion.dag_id == dag_id)).all())

assert pinned_id in remaining # still referenced by a task instance -> skipped
assert latest_id in remaining # kept by keep_last
assert orphan_id not in remaining # old and unreferenced -> pruned

def test_table_config_skip_if_referenced_requires_pk_column(self):
"""A misconfigured skip_if_referenced (pk not in columns) must fail fast at construction."""
with pytest.raises(ValueError, match="referenced_pk_column"):
_TableConfig(
table_name="dag_version",
recency_column_name="created_at",
dag_id_column_name="dag_id",
skip_if_referenced=[("task_instance", "dag_version_id")],
# "id" intentionally omitted from extra_columns
)

@patch("airflow.utils.db.reflect_tables")
def test_skip_archive_failure_will_remove_table(self, reflect_tables_mock):
"""
Expand Down
Loading