Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions airflow-core/src/airflow/models/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@

from typing import TYPE_CHECKING, Any

from sqlalchemy import Integer, MetaData, String, text
from sqlalchemy.orm import Mapped, mapped_column, registry
from sqlalchemy import Integer, MetaData, String, event, text
from sqlalchemy.orm import Mapped, Mapper, mapped_column, registry

from airflow.configuration import conf
from airflow.models import import_all_models

SQL_ALCHEMY_SCHEMA = conf.get("database", "SQL_ALCHEMY_SCHEMA")

Expand All @@ -48,6 +49,12 @@ def _get_schema():
mapper_registry = registry(metadata=metadata)
_sentinel = object()

# Models refer to each other by class name, such as relationship("DagBundleModel"), and a name resolves
# only once its module is imported. Import every model before SQLAlchemy first configures mappers,
# whether or not this process has a database engine. SQLAlchemy 2.0 offers this event only on the
# Mapper class (2.1 adds one per registry), so it fires on whichever registry configures first.
event.listen(Mapper, "before_configured", import_all_models, once=True)

if TYPE_CHECKING:
Base = Any
else:
Expand Down
4 changes: 0 additions & 4 deletions airflow-core/src/airflow/utils/orm_event_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from uuid import uuid4

from sqlalchemy import event, exc
from sqlalchemy.orm import Mapper

from airflow.configuration import conf

Expand All @@ -33,9 +32,6 @@

def setup_event_handlers(engine):
"""Setups event handlers."""
from airflow.models import import_all_models

event.listen(Mapper, "before_configured", import_all_models, once=True)

@event.listens_for(engine, "connect")
def connect(dbapi_connection, connection_record):
Expand Down
43 changes: 41 additions & 2 deletions airflow-core/tests/unit/models/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@
# under the License.
from __future__ import annotations

import os
import subprocess
import sys

import pytest

from airflow.models.base import get_id_collation_args

from tests_common.test_utils.config import conf_vars

pytestmark = pytest.mark.db_test


@pytest.mark.db_test
@pytest.mark.parametrize(
("dsn", "expected", "extra"),
[
Expand All @@ -50,3 +53,39 @@
def test_collation(dsn, expected, extra):
with conf_vars({("database", "sql_alchemy_conn"): dsn, **extra}):
assert expected == get_id_collation_args()


def test_models_configure_in_process_that_skips_orm_setup():
# Models name each other in string relationships, which resolve only once every model is imported.
# This process never calls configure_orm(), so building the first model must import the rest.
code = "from airflow.models.connection import Connection; Connection(conn_id='test', conn_type='http')"

result = subprocess.run(
[sys.executable, "-c", code],
env={**os.environ, "PYTHON_OPERATORS_VIRTUAL_ENV_MODE": "1"},
capture_output=True,
text=True,
check=False,
)

assert result.returncode == 0, result.stderr


def test_local_settings_can_build_models_while_airflow_is_imported(tmp_path):
# airflow_local_settings is imported before configure_orm() runs.
(tmp_path / "airflow_local_settings.py").write_text(
"from sqlalchemy.orm import configure_mappers\n"
"from airflow.models.team import Team\n"
"configure_mappers()\n"
)
python_path = os.pathsep.join(filter(None, [str(tmp_path), os.environ.get("PYTHONPATH")]))

result = subprocess.run(
[sys.executable, "-c", "import airflow"],
env={**os.environ, "PYTHONPATH": python_path},
capture_output=True,
text=True,
check=False,
)

assert result.returncode == 0, result.stderr
37 changes: 0 additions & 37 deletions airflow-core/tests/unit/utils/test_orm_event_handlers.py

This file was deleted.

15 changes: 0 additions & 15 deletions devel-common/src/tests_common/pytest_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,21 +139,6 @@
os.environ["AIRFLOW__DATABASE__SQL_ALCHEMY_CONN"] = "bad_schema:///"
# Set it here to pass the flag to python-xdist spawned processes
os.environ["_AIRFLOW_SKIP_DB_TESTS"] = "true"
# configure_orm() never succeeds against the connection string above, so the hook it installs to
# import every model before SQLAlchemy configures mappers is missing. Install the same hook here,
# or a string relationship such as Team.dag_bundles cannot resolve when a test first builds an ORM
# object. Imported lazily: importing airflow here would initialize its settings before this module
# finishes setting the environment, and not every distribution's test environment has SQLAlchemy.
if importlib.util.find_spec("sqlalchemy"):
from sqlalchemy import event
from sqlalchemy.orm import Mapper

def _import_all_models() -> None:
from airflow.models import import_all_models

import_all_models()

event.listen(Mapper, "before_configured", _import_all_models, once=True)

if run_db_tests_only:
# Set it here to pass the flag to python-xdist spawned processes
Expand Down
Loading