diff --git a/looptime/loops.py b/looptime/loops.py index fe4a6d2..45a3bac 100644 --- a/looptime/loops.py +++ b/looptime/loops.py @@ -1,10 +1,11 @@ from __future__ import annotations import asyncio +import contextlib import selectors import time import weakref -from typing import TYPE_CHECKING, Any, Callable, MutableSet, TypeVar, cast, overload +from typing import Iterator, TYPE_CHECKING, Any, Callable, MutableSet, TypeVar, cast, overload _T = TypeVar('_T') @@ -85,6 +86,11 @@ def setup_looptime( self.__sync_clock: Callable[[], float] = time.perf_counter self.__sync_ts: float | None = None # system/true-time clock timestamp + try: + self.__enabled # in case already entered via a ctx mgr — keep it + except AttributeError: + self.__enabled = 0 # if it is the 1st time, initialize as "off" + # TODO: why do we patch the selector as an object while the event loop as a class? # this should be the same patching method for both. try: @@ -93,6 +99,19 @@ def setup_looptime( self.__original_select = self._selector.select self._selector.select = self.__replaced_select # type: ignore + @contextlib.contextmanager + def looptime_enabled(self) -> Iterator[None]: + """ + Temporarily enable the time compaction, restore the normal mode on exit. + """ + if self.__enabled: + raise RuntimeError('Looptime mode is already enabled. Entered twice? Avoid this!') + self.__enabled += 1 + try: + yield + finally: + self.__enabled -= 1 + def time(self) -> float: return self.__int2time(self.__now) @@ -111,6 +130,16 @@ def __replaced_select(self, timeout: float | None) -> list[tuple[Any, Any]]: if ready: pass + # If nothing to do right now, and the time is not compacted, truly sleep as requested. + # Move the fake time by the exact real time spent in this wait (±discrepancies). + elif not self.__enabled: # reminder: this is a counter, but we care only about 0/non-0 + t0 = time.monotonic() + ready = self.__original_select(timeout=timeout) + t1 = time.monotonic() + + # If timeout=None, it never exists until ready. This timeout check is for typing only. + self.__now += self.__time2int(t1 - t0 if ready or timeout is None else timeout) + # Regardless of the timeout, if there are executors sync futures, we move the time in steps. # The timeout (if present) can limit the size of the step, but not the logic of stepping. # Generally, external things (threads) take some time (e.g. for thread spawning). diff --git a/looptime/plugin.py b/looptime/plugin.py index 6b6405a..997c266 100644 --- a/looptime/plugin.py +++ b/looptime/plugin.py @@ -1,3 +1,106 @@ +""" +Integrations with pytest & pytest-asyncio (irrelevant for other frameworks). + +The critical implementation details and the rationale (re-read before changes): + + +PROBLEM +======= + +Pytest-asyncio>=1.0.0 has removed the ``event_loop`` fixture and fully switched +to the ``event_loop_policy`` (session-scoped) plus several independent fixtures: +session-, packages-, module-, class-, function-scoped. It means that a test +might use any of these fixtures. + +As a result, our previous assumption that every test runs in its own event loop +is broken. As such, an instance of an event loop can be shared by many tests. + +Time, by its nature, MUST be monotonic (always growing, never going backwards). +If we break this core assumption, all hell breaks loose. An anit-example: +the callbacks and other events triggering before they were set up (clock-wise), +the durations of activities being negative, so on. We simply do not do that. + +Therefore, we cannot reset the time to zero for every test as we did before. +Therefore, the 2nd, 3rd, so on tests do not start at the loop time "zero", +but at the ever-increasing clock value. + +The looptime library, however, is made to simplify the loop time measurements +within a single test (in assertions). This intention comes into a conflict +with the new concept of shared event loops. + + +SOLUTION +======== + +In order to solve the conceptual conflict, we abandon the assumption that time +of an event loop is zero-based per test (it can be zero-based per loop though). + +Instead, we double-down on the assumption that the ``looptime`` fixture +measures the time on a per-test level and should be used in assertions +(previously, is was a synonym for ``asyncio.get_running_loop().time()``):: + + async def test_me(looptime): + await asyncio.sleep(123) + assert looptime == 123 + +This, in turn, brings a few consequences to the implementation: + + +CONSEQUENCE 1 — inverted code flow +================================== + +It might seem that the easiest way to implement the ``looptime`` fixture is +to make it ``async def`` and get the running loop inside. This does NOT work. + +When a function-scoped fixture is used in any higher-scoped test, it degrades +the test from its scope to the function scope and breaks the test design. +See an example at https://github.com/pytest-dev/pytest-asyncio/issues/1142. + +As such, the fixture MUST be synchronous (simple ``def``). As a result, +the fixture CANNOT get a running loop, because there is no running loop. + +However, the intended event loop is available in the test hook. But the tricky +part is that fixtures are set up _outside_ (i.e. before) the test hook. So, +the inner-nested hook should pass the data into the outer-nested fixture object. + +For this, we use pytest stashes (any arbitrary mutable object/dict would work): + +- The fixture, when created, remembers the stash. +- The hook populates the stash with the "proper" loop and/or start time. +- The fixture, when evaluated, looks into that stash and gets its value. + +Note that the "proper" loop can be of any scope as designed by the test authors +and does not degrade the test to the function-scoped event loop anymore. + +There is no easy way how this sophisticated design can be simplified. + + +CONSEQUENCE 2 — on-demand time compaction +========================================= + +In order to make event loops compatible with looptime, they (the event loops) +MUST be patched at creation, not in the middle of a runtime when it hits +the looptime-enabled tests (consider a global session-scoped event loop here). + +First of all, we now patch not the event loop, but the event loop policy, since +this is the only publicly documented fixture and the source of event loops. +The patched event loop policy simply produces the patched event loops. + +However, tests may be designed either for the "true" time or the "fake" time, +intermixed in any order. We should compact the time only if and when requested. + +So, even with the monkey-patched event loop and event loop policy, we toggle off +the time magic by default, and toggle it on for the tests marked for looptime. + + +RESULT +====== + +With these hack incapsulated in the looptime library, the time compaction works +as it was intended: only when and if requested, with the time measured per test, +while still supporting the new pytest-asyncio's multi-scoped event loops. +""" + from __future__ import annotations import asyncio @@ -5,11 +108,25 @@ import pytest -from looptime import loops, patchers +from looptime import loops, patchers, timeproxies, policies + + +def pytest_configure(config: Any) -> None: + config.addinivalue_line('markers', "looptime: configure the fake fast-forwarding loop time.") + + +def pytest_addoption(parser: Any) -> None: + group = parser.getgroup("asyncio time contraction") + group.addoption("--no-looptime", dest='looptime', action="store_const", const=False, + help="Force all (even marked) tests to the true loop time.") + group.addoption("--looptime", dest='looptime', action="store_const", const=True, + help="Run unmarked tests with the fake loop time by default.") @pytest.hookimpl(hookwrapper=True) def pytest_fixture_setup(fixturedef: Any, request: Any) -> Any: + # pytest-asyncio<1.0.0 exposed the specific "event_loop" fixture; deprecated since >=0.23.0. + # But we still support for the older versions, or if some other plugins provide it. if fixturedef.argname == "event_loop": result = yield loop = cast(asyncio.BaseEventLoop, result.get_result()) if result.excinfo is None else None @@ -29,17 +146,121 @@ def pytest_fixture_setup(fixturedef: Any, request: Any) -> Any: patched_loop = patchers.patch_event_loop(loop) patched_loop.setup_looptime(**options) result.force_result(patched_loop) + + # pytest-asyncio>=1.0.0 exposes only the "event_loop_policy"; available since >=0.23.0. + # Always patch the whole policy, always at creation. But toggle the magic on & off when needed. + elif fixturedef.argname == "event_loop_policy": + result = yield + policy = cast(asyncio.AbstractEventLoopPolicy, result.get_result()) if result.excinfo is None else None + if policy is not None and not isinstance(policy, policies.LoopTimeEventLoopPolicy): + + # True means implicitly on; False means explicitly off; None means "only if marked". + option: bool | None = request.config.getoption('looptime') + enabled = bool(option is not False) # None means "maybe", so still patch it. + if enabled: + patched_policy = policies.patch_event_loop_policy(policy) + result.force_result(patched_policy) + else: yield -def pytest_configure(config: Any) -> None: - config.addinivalue_line('markers', "looptime: configure the fake fast-forwarding loop time.") +# NB: It MUST be sync! It CANNOT be async — see the module's docstring. +@pytest.fixture +def looptime(request: pytest.FixtureRequest) -> timeproxies.LoopTimeProxy: + """ + Expose the time of the test run in the loop-clock time. + The fixture's value is the number of seconds since the start of the test: -def pytest_addoption(parser: Any) -> None: - group = parser.getgroup("asyncio time contraction") - group.addoption("--no-looptime", dest='looptime', action="store_const", const=False, - help="Force all (even marked) tests to the true loop time.") - group.addoption("--looptime", dest='looptime', action="store_const", const=True, - help="Run unmarked tests with the fake loop time by default.") + - It can be used in assertions & comparisons (``==``, ``<=``, etc). + - It can also be used in simple math (additions, substractions, etc). + - It can be converted to ``int()`` or ``float()``. + + The assumption is that typical fixtures do not take the loop time, + i.e. have no intentional sleeps or delays (the external i/o is not counted). + The fixtures should prepare the environment, the test does the timed things. + + If fixtures do introduce delays, make sure they depend on this fixture, + so that their time spent is counted towards the fixture's numeric value. + + To make it clear: the fixture assumes that the "time zero" is the moment + when the test function was entered, as seen by the event loop's time. + The event loop's time can be zero for the function-scoped event loops, + but it can also be any arbitrary monotonic value for event loops shared + by multiple tests with wider scopes (class, module, package, session). + """ + # Note: the proper "time zero" is NOT yet set. This happens in the hook below — after this line. + return timeproxies.LoopTimeProxy() + + +# This hook is the latest (deepest) possible entrypoint before diving into the test function itself, +# with all the fixtures executed earlier, so that their setup time is not taken into account. +# By design, the `looptime` fixture should indicate ONLY the runtime of the test itself. +# The alternatives to consider — the subtle differences are unclear for now: +# - pytest_pyfunc_call(pyfuncitem) +# - pytest_runtest_call(item) +# - pytest_runtest_setup(item), wrapped as @hookimpl(trylast=True) +# But only pytest_fixture_setup(fixturedef, request) has the documented `request` for used fixtures. +@pytest.hookimpl(wrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function) -> Any: + + # Get the policy from the pre-populated & pre-resolved fixture values (done in the setup stage). + # This includes all the auto-used fixtures, but NOT the dynamic `getfixturevalue(…)` ones. + # Alternatively, use the private `pyfuncitem._request.getfixturevalue(…)`, though this is hacky. + funcargs: dict[str, Any] = pyfuncitem.funcargs + + # Not pytest-asyncio-enabled? Then let it run somehow as usual — not our business. + if 'event_loop_policy' not in funcargs: + return (yield) + + # Important: this can be ANY event loop of ANY declared scope of pytest-asyncio. + # The hook itself has NO "running" loop (because it is sync, not async). + policy: asyncio.AbstractEventLoopPolicy = funcargs['event_loop_policy'] + loop = policy.get_event_loop() + print(f"HOOK {id(loop)=} {loop=}") # TODO do not merge + + # The event loop is not patched, we are doomed to fail, so let it run somehow on its own. + # This might happen if the custom event loop policy explicitly produces incompatible loops. + if not isinstance(loop, loops.LoopTimeEventLoop): + return (yield) + + # True means implicitly on; False means explicitly off; None means "only if marked". + option: bool | None = pyfuncitem.config.getoption('looptime') + globally_disabled = option is False # but not None! + globally_enforced = option is True + + # Decide on the test's intentions: looptime-enabled or now, which options, etc. + markers = list(pyfuncitem.iter_markers('looptime')) + enabled = bool((markers or globally_enforced) and not globally_disabled) + options: dict[str, Any] = {} + for marker in reversed(markers): + options.update(marker.kwargs) + enabled = bool(marker.args[0]) if marker.args else enabled + + # If not enabled/enforced for this test, even if the event loop is patched, let it run as usual. + if not enabled: + return (yield) + + # Finally, if enabled/enforced, configure and run the test in the compacted time mode. + # Note: The loop's time cannot be moved backwards (see the module docstring). + # So, peg it at the current time, but adjust the looptime fixture to reflect the start=… kwarg. + # TODO: except start=? end=? or adjusted to the current values? + desired_start = options.pop('start', None) + desired_end = options.pop('end', None) + loop_now = loop.time() + + # Adjust the start/end time to move the time monotonically as explained in the docstring. + # Technically, we can reset it to zero on every test, but the consequences are unpredictable. + options['start'] = loop_now + options['end'] = loop_now + desired_end if desired_end is not None else None # TODO: callables/clocks + + # Set the "time zero" in the ``looptime`` fixture (again). NB: Fixtures are set up outside + # of the test/hook, some fixtures can take some time, so this hook's time is the most precise. + if 'looptime' in funcargs: + looptime: timeproxies.LoopTimeProxy = funcargs['looptime'] + looptime.zero = loop_now - (desired_start or 0) + + with loop.looptime_enabled(): + loop.setup_looptime(**options) + return (yield) diff --git a/looptime/policies.py b/looptime/policies.py new file mode 100644 index 0000000..9405169 --- /dev/null +++ b/looptime/policies.py @@ -0,0 +1,112 @@ +import asyncio +from typing import Callable, Any, Type, cast + +from looptime import loops, patchers + + +# TODO: BaseDefaultEventLoopPolicy or AbstractDefaultEventLoopPolicy for a mixin? +class LoopTimeEventLoopPolicy(asyncio.DefaultEventLoopPolicy): + """ + A mixin to inject into event loop policies to make them looptime-enabled. + + This policy mixin ensures that all event loops produced are either already + looptime-enabled (i.e. inherit from :class:`LoopTimeEventLoop`), + or it monkey-patches the newly produced event loops to be looptime-enabled. + + This mixin can be used explicitly when defining the custom policy classes. + It is used implicitly when monkey-patching the existing policies when + enforcing the looptime capabilities in tests with pytest-asyncio>=1.0.0. + + For monkey-patching, a new empty (no-member) class is created at runtime, + with tis mixin and the original policy class as the bases, and is injected + into the policy's instance ``__class__`` attribute. + """ + + # Precisely the args/kwargs as in the LoopTimeEventLoop's constructor. + # Args/kwargs are passed through in case this class is used as a mixin. + def __init__( + self, + *args: Any, + start: float | None | Callable[[], float | None] = None, + end: float | None | Callable[[], float | None] = None, + resolution: float = 1e-6, # to cut off the floating-point errors + idle_step: float | None = None, + idle_timeout: float | None = None, + noop_cycles: int = 42, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self.setup_looptime( + start=start, + end=end, + resolution=resolution, + idle_step=idle_step, + idle_timeout=idle_timeout, + noop_cycles=noop_cycles, + ) + + def setup_looptime( + self, + *, + start: float | None | Callable[[], float | None] = None, + end: float | None | Callable[[], float | None] = None, + resolution: float = 1e-6, # to cut off the floating-point errors + idle_step: float | None = None, + idle_timeout: float | None = None, + noop_cycles: int = 42, + ) -> None: + self.__start = start + self.__end = end + self.__resolution = resolution + self.__idle_step = idle_step + self.__idle_timeout = idle_timeout + self.__noop_cycles = noop_cycles + + # TODO: which method? + # def get_event_loop(self) -> loops.LoopTimeEventLoop: + def new_event_loop(self) -> loops.LoopTimeEventLoop: + # loop = super().get_event_loop() + loop = super().new_event_loop() + return patchers.patch_event_loop( + loop, + start=self.__start, + end=self.__end, + resolution=self.__resolution, + idle_step=self.__idle_step, + idle_timeout=self.__idle_timeout, + noop_cycles=self.__noop_cycles, + ) + + +_policies_cache: dict[Type[asyncio.AbstractEventLoopPolicy], Type[LoopTimeEventLoopPolicy]] = {} + + +def make_event_loop_policy_class( + cls: Type[asyncio.AbstractEventLoopPolicy], + *, + prefix: str = 'Looptime', +) -> type[LoopTimeEventLoopPolicy]: + if issubclass(cls, LoopTimeEventLoopPolicy): + return cls + elif cls not in _policies_cache: + new_class = type(f'{prefix}{cls.__name__}', (LoopTimeEventLoopPolicy, cls), {}) + _policies_cache[cls] = new_class + return _policies_cache[cls] + + +def patch_event_loop_policy( + policy: asyncio.AbstractEventLoopPolicy, + **kwargs: Any, +) -> LoopTimeEventLoopPolicy: + """ + Convert any pre-existing event loop policy to be looptime-enabled. + """ + result: loops.LoopTimeEventLoop + if isinstance(policy, LoopTimeEventLoopPolicy): + return policy + else: + new_class: type[LoopTimeEventLoopPolicy] = make_event_loop_policy_class(policy.__class__) + policy.__class__ = new_class + policy = cast(LoopTimeEventLoopPolicy, policy) + policy.setup_looptime(**kwargs) + return policy diff --git a/looptime/timeproxies.py b/looptime/timeproxies.py index 7886df2..6fe1afd 100644 --- a/looptime/timeproxies.py +++ b/looptime/timeproxies.py @@ -9,6 +9,7 @@ class LoopTimeProxy(math.Numeric): """ A numeric-compatible proxy to the time of the current/specific event loop. """ + zero: float # mutable! where our "time zero" is in the monotonic loop time. def __init__( self, @@ -18,6 +19,7 @@ def __init__( ) -> None: super().__init__(resolution=resolution) self._loop = loop + self.zero = 0 def __repr__(self) -> str: return f"" @@ -30,14 +32,5 @@ def __matmul__(self, other: object) -> LoopTimeProxy: @property def _value(self) -> float: - return self._loop.time() if self._loop is not None else asyncio.get_running_loop().time() - - -try: - import pytest -except ImportError: - pass -else: - @pytest.fixture() - def looptime() -> LoopTimeProxy: - return LoopTimeProxy() + loop = self._loop if self._loop is not None else asyncio.get_running_loop() + return loop.time() - self.zero diff --git a/tests/test_chronometers.py b/tests/test_chronometers.py index ae6a2ed..8597a5d 100644 --- a/tests/test_chronometers.py +++ b/tests/test_chronometers.py @@ -52,7 +52,8 @@ async def test_async_context_manager(): @pytest.mark.asyncio @pytest.mark.looptime(start=100) -async def test_readme_example(chronometer, event_loop): +async def test_readme_example(chronometer): + event_loop = asyncio.get_running_loop() with chronometer, looptime.Chronometer(event_loop.time) as loopometer: await asyncio.sleep(1) await asyncio.sleep(1) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 2d30b8e..071db17 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3,28 +3,33 @@ def test_cli_default_mode(testdir): testdir.makepyfile(""" + import asyncio import looptime import pytest @pytest.mark.asyncio - async def test_normal(event_loop): + async def test_normal(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) @pytest.mark.asyncio @pytest.mark.looptime - async def test_marked(event_loop): + async def test_marked(): + event_loop = asyncio.get_running_loop() assert isinstance(event_loop, looptime.LoopTimeEventLoop) assert event_loop.time() == 0 @pytest.mark.asyncio @pytest.mark.looptime(start=123) - async def test_configured(event_loop): + async def test_configured(): + event_loop = asyncio.get_running_loop() assert isinstance(event_loop, looptime.LoopTimeEventLoop) assert event_loop.time() == 123 @pytest.mark.asyncio @pytest.mark.looptime(False) - async def test_disabled(event_loop): + async def test_disabled(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) """) result = testdir.runpytest() @@ -33,29 +38,34 @@ async def test_disabled(event_loop): def test_cli_option_enabled(testdir): testdir.makepyfile(""" + import asyncio import looptime import pytest @pytest.mark.asyncio - async def test_normal(event_loop): + async def test_normal(): + event_loop = asyncio.get_running_loop() assert isinstance(event_loop, looptime.LoopTimeEventLoop) assert event_loop.time() == 0 @pytest.mark.asyncio @pytest.mark.looptime - async def test_marked(event_loop): + async def test_marked(): + event_loop = asyncio.get_running_loop() assert isinstance(event_loop, looptime.LoopTimeEventLoop) assert event_loop.time() == 0 @pytest.mark.asyncio @pytest.mark.looptime(start=123) - async def test_configured(event_loop): + async def test_configured(): + event_loop = asyncio.get_running_loop() assert isinstance(event_loop, looptime.LoopTimeEventLoop) assert event_loop.time() == 123 @pytest.mark.asyncio @pytest.mark.looptime(False) - async def test_disabled(event_loop): + async def test_disabled(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) """) result = testdir.runpytest("--looptime") @@ -64,26 +74,31 @@ async def test_disabled(event_loop): def test_cli_option_disabled(testdir): testdir.makepyfile(""" + import asyncio import looptime import pytest @pytest.mark.asyncio - async def test_normal(event_loop): + async def test_normal(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) @pytest.mark.asyncio @pytest.mark.looptime - async def test_marked(event_loop): + async def test_marked(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) @pytest.mark.asyncio @pytest.mark.looptime(start=123) - async def test_configured(event_loop): + async def test_configured(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) @pytest.mark.asyncio @pytest.mark.looptime(False) - async def test_disabled(event_loop): + async def test_disabled(): + event_loop = asyncio.get_running_loop() assert not isinstance(event_loop, looptime.LoopTimeEventLoop) """) result = testdir.runpytest("--no-looptime")