From 12a54d29f17477ae04b85b762e3308e2e7a42b64 Mon Sep 17 00:00:00 2001 From: Matthias Wende Date: Fri, 1 Dec 2023 11:18:27 +0100 Subject: [PATCH 01/32] Expose resampling-related exceptions publicly We expose `ResamplingError` and `SourceStoppedError` as users might need to catch these errors. Signed-off-by: Leandro Lucarella --- src/frequenz/sdk/timeseries/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/frequenz/sdk/timeseries/__init__.py b/src/frequenz/sdk/timeseries/__init__.py index 97af5a478..6ec77000c 100644 --- a/src/frequenz/sdk/timeseries/__init__.py +++ b/src/frequenz/sdk/timeseries/__init__.py @@ -42,16 +42,19 @@ from ._periodic_feature_extractor import PeriodicFeatureExtractor from ._resampling._base_types import SourceProperties from ._resampling._config import ResamplerConfig, ResamplingFunction +from ._resampling._exceptions import ResamplingError, SourceStoppedError __all__ = [ "Bounds", "Fuse", "MovingWindow", "PeriodicFeatureExtractor", - "ResamplerConfig", "ReceiverFetcher", + "ResamplerConfig", + "ResamplingError", "ResamplingFunction", "Sample", "Sample3Phase", "SourceProperties", + "SourceStoppedError", ] From 9da8d454d567518db5c590beb7e06ff3583401a8 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Tue, 29 Jul 2025 11:43:56 +0200 Subject: [PATCH 02/32] Add a wall clock attached timer for the resampler This timer uses the wall clock to trigger ticks and handles discrepancies between the wall clock and monotonic time. Since sleeping is performed using monotonic time, differences between the two clocks can occur. When the wall clock progresses slower than monotonic time, it is referred to as *compression* (wall clock time appears in the past relative to monotonic time). Conversely, when the wall clock progresses faster, it is called *expansion* (wall clock time appears in the future relative to monotonic time). If these differences exceed a configured threshold, a warning is emitted. If the difference becomes excessively large, it is treated as a *time jump*. Time jumps can occur, for example, when the wall clock is adjusted by NTP after being out of sync for an extended period. In such cases, the timer resynchronizes with the wall clock and triggers an immediate tick. Signed-off-by: Leandro Lucarella --- src/frequenz/sdk/timeseries/__init__.py | 10 + .../_resampling/_wall_clock_timer.py | 849 ++++++++++++++++++ 2 files changed, 859 insertions(+) create mode 100644 src/frequenz/sdk/timeseries/_resampling/_wall_clock_timer.py diff --git a/src/frequenz/sdk/timeseries/__init__.py b/src/frequenz/sdk/timeseries/__init__.py index 6ec77000c..c57bbaa5e 100644 --- a/src/frequenz/sdk/timeseries/__init__.py +++ b/src/frequenz/sdk/timeseries/__init__.py @@ -43,9 +43,16 @@ from ._resampling._base_types import SourceProperties from ._resampling._config import ResamplerConfig, ResamplingFunction from ._resampling._exceptions import ResamplingError, SourceStoppedError +from ._resampling._wall_clock_timer import ( + ClocksInfo, + TickInfo, + WallClockTimer, + WallClockTimerConfig, +) __all__ = [ "Bounds", + "ClocksInfo", "Fuse", "MovingWindow", "PeriodicFeatureExtractor", @@ -57,4 +64,7 @@ "Sample3Phase", "SourceProperties", "SourceStoppedError", + "TickInfo", + "WallClockTimer", + "WallClockTimerConfig", ] diff --git a/src/frequenz/sdk/timeseries/_resampling/_wall_clock_timer.py b/src/frequenz/sdk/timeseries/_resampling/_wall_clock_timer.py new file mode 100644 index 000000000..2b1ac2837 --- /dev/null +++ b/src/frequenz/sdk/timeseries/_resampling/_wall_clock_timer.py @@ -0,0 +1,849 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""A timer attached to the wall clock for the resampler.""" + +from __future__ import annotations + +import asyncio +import logging +import math +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Literal, Self, assert_never + +from frequenz.channels import Receiver, ReceiverStoppedError +from frequenz.core.datetime import UNIX_EPOCH +from typing_extensions import override + +_logger = logging.getLogger(__name__) + +_TD_ZERO = timedelta() + + +@dataclass(frozen=True, kw_only=True) +class WallClockTimerConfig: + """Configuration for a [wall clock timer][frequenz.sdk.timeseries.WallClockTimer].""" + + align_to: datetime | None = UNIX_EPOCH + """The time to align the timer to. + + The first timer tick will occur at the first multiple of the timer's interval after + this value. + + It must be a timezone aware `datetime` or `None`. If `None`, the timer aligns to the + time it is started. + """ + + async_drift_tolerance: timedelta | None = None + """The maximum allowed difference between the requested and the real sleep time. + + The timer will emit a warning if the difference is bigger than this value. + + It must be bigger than 0 or `None`. If `None`, no warnings will ever be emitted. + """ + + wall_clock_drift_tolerance_factor: float | None = None + """The maximum allowed relative difference between the wall clock and monotonic time. + + The timer will emit a warning if the relative difference is bigger than this value. + If the difference remains constant, the warning will be emitted only once, as the + previous drift is taken into account. If there is information on the previous drift, + the previous and current factor will be used to determine if a warning should be + emitted. + + It must be bigger than 0 or `None`. If `None`, no warnings will be ever emitted. + + Info: + The calculation is as follows: + + ```py + tolerance = wall_clock_drift_tolerance_factor + factor = monotonic_elapsed / wall_clock_elapsed + previous_factor = previous_monotonic_elapsed / previous_wall_clock_elapsed + if abs(factor - previous_factor) > tolerance: + emit warning + ``` + + If there is no previous information, a `previous_factor` of 1.0 will be used. + """ + + wall_clock_jump_threshold: timedelta | None = None + """The amount of time that's considered a wall clock jump. + + When the drift between the wall clock and monotonic time is too big, it is + considered a time jump and the timer will be resynced to the wall clock. + + This value determines how big the difference needs to be to be considered a + jump. + + Smaller values are considered wall clock *expansions* or *compressions* and are + always gradually adjusted, instead of triggering a resync. + + Must be bigger than 0 or `None`. If `None`, a resync will never be triggered due to + time jumps. + """ + + def __post_init__(self) -> None: + """Check that config values are valid. + + Raises: + ValueError: If any value is out of range. + """ + if self.align_to is not None and self.align_to.tzinfo is None: + raise ValueError( + f"align_to ({self.align_to}) should be a timezone aware datetime" + ) + + def _is_strictly_positive_or_none(value: float | timedelta | None) -> bool: + match value: + case None: + return True + case timedelta() as delta: + return delta > _TD_ZERO + case float() as num: + return math.isfinite(num) and num > 0.0 + case int() as num: + return num > 0 + case _ as unknown: + assert_never(unknown) + + if not _is_strictly_positive_or_none(self.async_drift_tolerance): + raise ValueError( + "async_drift_tolerance should be positive or None, not " + f"{self.async_drift_tolerance!r}" + ) + if not _is_strictly_positive_or_none(self.wall_clock_drift_tolerance_factor): + raise ValueError( + "wall_clock_drift_tolerance_factor should be positive or None, not " + f"{self.wall_clock_drift_tolerance_factor!r}" + ) + if not _is_strictly_positive_or_none(self.wall_clock_jump_threshold): + raise ValueError( + "wall_clock_jump_threshold should be positive or None, not " + f"{self.wall_clock_jump_threshold!r}" + ) + + @classmethod + def from_interval( # pylint: disable=too-many-arguments + cls, + interval: timedelta, + *, + align_to: datetime | None = UNIX_EPOCH, + async_drift_tolerance_factor: float = 0.1, + wall_clock_drift_tolerance_factor: float = 0.1, + wall_clock_jump_threshold_factor: float = 1.0, + ) -> Self: + """Create a timer configuration based on an interval. + + This will set the tolerance and threshold values proportionally to the interval. + + Args: + interval: The interval between timer ticks. Must be bigger than 0. + align_to: The time to align the timer to. See the + [`WallClockTimer`][frequenz.sdk.timeseries.WallClockTimer] documentation + for details. + async_drift_tolerance_factor: The maximum allowed difference between the + requested and the real sleep time, relative to the interval. + `async_drift_tolerance` will be set to `interval * this_factor`. See + the [`WallClockTimer`][frequenz.sdk.timeseries.WallClockTimer] + documentation for details. + wall_clock_drift_tolerance_factor: The maximum allowed relative difference + between the wall clock and monotonic time. See the + [`WallClockTimer`][frequenz.sdk.timeseries.WallClockTimer] documentation + for details. + wall_clock_jump_threshold_factor: The amount of time that's considered a + wall clock jump, relative to the interval. This will be set to + `interval * this_factor`. See the + [`WallClockTimer`][frequenz.sdk.timeseries.WallClockTimer] documentation + for details. + + Returns: + The created timer configuration. + + Raises: + ValueError: If any value is out of range. + """ + if interval <= _TD_ZERO: + raise ValueError(f"interval must be bigger than 0, not {interval!r}") + + return cls( + align_to=align_to, + wall_clock_drift_tolerance_factor=wall_clock_drift_tolerance_factor, + async_drift_tolerance=interval * async_drift_tolerance_factor, + wall_clock_jump_threshold=interval * wall_clock_jump_threshold_factor, + ) + + +@dataclass(frozen=True, kw_only=True) +class ClocksInfo: + """Information about the wall clock and monotonic clock and their drift. + + The `monotonic_requested_sleep` and `monotonic_elapsed` values must be strictly + positive, while the `wall_clock_elapsed` can be negative if the wall clock jumped + back in time. + """ + + monotonic_requested_sleep: timedelta + """The requested monotonic sleep time used to gather the information (must be positive).""" + + monotonic_time: float + """The monotonic time right after the sleep was done.""" + + wall_clock_time: datetime + """The wall clock time right after the sleep was done.""" + + monotonic_elapsed: timedelta + """The elapsed time in monotonic time (must be non-negative).""" + + wall_clock_elapsed: timedelta + """The elapsed time in wall clock time.""" + + wall_clock_factor: float = float("nan") + """The factor to convert wall clock time to monotonic time. + + Typically, if the wall clock time expanded compared to the monotonic time (i.e. + is more in the future), the returned value will be smaller than 1. If the wall + clock time compressed compared to the monotonic time (i.e. is more in the past), + the returned value will be bigger than 1. + + In cases where there are big time jumps this might be overridden by the previous + wall clock factor to avoid adjusting by excessive amounts, when the time will + resync anyway to catch up. + """ + + def __post_init__(self) -> None: + """Check that the values are valid. + + Raises: + ValueError: If any value is out of range. + """ + if self.monotonic_requested_sleep <= _TD_ZERO: + raise ValueError( + f"monotonic_requested_sleep must be strictly positive, not " + f"{self.monotonic_requested_sleep!r}" + ) + if not math.isfinite(self.monotonic_time): + raise ValueError( + f"monotonic_time must be a number, not {self.monotonic_time!r}" + ) + if self.monotonic_elapsed <= _TD_ZERO: + raise ValueError( + f"monotonic_elapsed must be strictly positive, not {self.monotonic_elapsed!r}" + ) + + # This is a hack to cache the calculated value, once set it will be "immutable" + # too, so it shouldn't change the logical "frozenness" of the class. + if math.isnan(self.wall_clock_factor): + wall_clock_elapsed = self.wall_clock_elapsed + if wall_clock_elapsed <= _TD_ZERO: + _logger.warning( + "The monotonic clock advanced %s, but the wall clock stayed still or " + "jumped back (elapsed: %s)! Hopefully this was just a singular jump in " + "time and not a permanent issue with the wall clock not moving at all. " + "For purposes of calculating the wall clock factor, a fake elapsed time " + "of one tenth of the elapsed monotonic time will be used.", + self.monotonic_elapsed, + wall_clock_elapsed, + ) + wall_clock_elapsed = self.monotonic_elapsed * 0.1 + # We need to use __setattr__ here to bypass the frozen nature of the + # dataclass. Since we are constructing the class, this is fine and the only + # way to set calculated defaults in frozen dataclasses at the moment. + super().__setattr__( + "wall_clock_factor", self.monotonic_elapsed / wall_clock_elapsed + ) + + @property + def monotonic_drift(self) -> timedelta: + """The difference between the monotonic elapsed and requested sleep time. + + This number should be always positive, as the monotonic time should never + jump back in time. + """ + return self.monotonic_elapsed - self.monotonic_requested_sleep + + @property + def wall_clock_jump(self) -> timedelta: + """The amount of time the wall clock jumped compared to the monotonic time. + + If the wall clock is faster then the monotonic time (or jumped forward in time), + the returned value will be positive. If the wall clock is slower than the + monotonic time (or jumped backwards in time), the returned value will be + negative. + + Note: + Strictly speaking, both could be in sync and the result would be 0.0, but + this is extremely unlikely due to floating point precision and the fact + that both clocks are obtained as slightly different times. + """ + return self.wall_clock_elapsed - self.monotonic_elapsed + + def wall_clock_to_monotonic(self, wall_clock_timedelta: timedelta, /) -> timedelta: + """Convert a wall clock timedelta to a monotonic timedelta. + + This is useful to calculate how much one should sleep on the monotonic clock + to reach a particular wall clock time, adjusting to the difference in speed + or jumps between both. + + Args: + wall_clock_timedelta: The wall clock timedelta to convert. + + Returns: + The monotonic timedelta corresponding to `wall_clock_time` using the + `wall_clock_factor`. + """ + return wall_clock_timedelta * self.wall_clock_factor + + +@dataclass(frozen=True, kw_only=True) +class TickInfo: + """Information about a `WallClockTimer` tick.""" + + expected_tick_time: datetime + """The expected time when the timer should have triggered.""" + + sleep_infos: Sequence[ClocksInfo] + """The information about every sleep performed to trigger this tick. + + If the timer didn't have do to a [`sleep()`][asyncio.sleep] to trigger the tick + (i.e. the timer is catching up because there were big drifts in previous ticks), + this will be empty. + """ + + @property + def latest_sleep_info(self) -> ClocksInfo | None: + """The clocks information from the last sleep done to trigger this tick. + + If no sleeps were done, this will be `None`. + """ + return self.sleep_infos[-1] if self.sleep_infos else None + + +class WallClockTimer(Receiver[TickInfo]): + """A timer synchronized with the wall clock. + + This timer uses the wall clock to trigger ticks and handles discrepancies between + the wall clock and monotonic time. Since sleeping is performed using monotonic time, + differences between the two clocks can occur. + + When the wall clock progresses slower than monotonic time, it is referred to as + *compression* (wall clock time appears in the past relative to monotonic time). + Conversely, when the wall clock progresses faster, it is called *expansion* + (wall clock time appears in the future relative to monotonic time). If these + differences exceed a configured threshold, a warning is emitted. The threshold + is defined by the + [`wall_clock_drift_tolerance_factor`][frequenz.sdk.timeseries.WallClockTimerConfig.wall_clock_drift_tolerance_factor]. + + If the difference becomes excessively large, it is treated as a *time jump*. + Time jumps can occur, for example, when the wall clock is adjusted by NTP after + being out of sync for an extended period. In such cases, the timer resynchronizes + with the wall clock and triggers an immediate tick. The threshold for detecting + time jumps is controlled by the + [`wall_clock_jump_threshold`][frequenz.sdk.timeseries.WallClockTimerConfig.wall_clock_jump_threshold]. + + The timer ensures ticks are aligned to the + [`align_to`][frequenz.sdk.timeseries.WallClockTimerConfig.align_to] configuration, + even after time jumps. + + Additionally, the timer emits warnings if the actual sleep duration deviates + significantly from the requested duration. This can happen due to event loop + blocking or system overload. The tolerance for such deviations is defined by the + [`async_drift_tolerance`][frequenz.sdk.timeseries.WallClockTimerConfig.async_drift_tolerance]. + + To account for these complexities, each tick provides a + [`TickInfo`][frequenz.sdk.timeseries.TickInfo] object, which includes detailed + information about the clocks and their drift. + """ + + def __init__( + self, + interval: timedelta, + config: WallClockTimerConfig | None = None, + *, + auto_start: bool = True, + ) -> None: + """Initialize this timer. + + See the class documentation for details. + + Args: + interval: The time between timer ticks. Must be positive. + config: The configuration for the timer. If `None`, a default configuration + will be created using `from_interval()`. + auto_start: Whether the timer should start automatically. If `False`, + `reset()` must be called before the timer can be used. + + Raises: + ValueError: If any value is out of range. + """ + if interval <= _TD_ZERO: + raise ValueError(f"interval must be positive, not {interval}") + + self._interval: timedelta = interval + """The time to between timer ticks. + + The wall clock is used, so this will be added to the current time to calculate + the next tick time. + """ + + self._config = config or WallClockTimerConfig.from_interval(interval) + """The configuration for this timer.""" + + self._closed: bool = True + """Whether the timer was requested to close. + + If this is `False`, then the timer is running. + + If this is `True`, then it is closed or there is a request to close it + or it was not started yet: + + * If `_next_tick_time` is `None`, it means it wasn't started yet (it was + created with `auto_start=False`). Any receiving method will start + it by calling `reset()` in this case. + + * If `_next_tick_time` is not `None`, it means there was a request to + close it. In this case receiving methods will raise + a `ReceiverStoppedError`. + """ + + self._next_tick_time: datetime | None = None + """The wall clock time when the next tick should happen. + + If this is `None`, it means the timer didn't start yet, but it should + be started as soon as it is used. + """ + + self._current_tick_info: TickInfo | None = None + """The current tick information. + + This is calculated by `ready()` but is returned by `consume()`. If + `None` it means `ready()` wasn't called and `consume()` will assert. + `consume()` will set it back to `None` to tell `ready()` that it needs + to wait again. + """ + + self._clocks_info: ClocksInfo | None = None + """The latest information about the clocks and their drift. + + Or `None` if no sleeps were done yet. + """ + + if auto_start: + self.reset() + + @property + def interval(self) -> timedelta: + """The interval between timer ticks. + + Since the wall clock is used, this will be added to the current time to + calculate the next tick time. + + Danger: + In real (monotonic) time, the actual time it passes between ticks could be + smaller, bigger, or even **negative** if the wall clock jumped back in time! + """ + return self._interval + + @property + def config(self) -> WallClockTimerConfig: + """The configuration for this timer.""" + return self._config + + @property + def is_running(self) -> bool: + """Whether the timer is running.""" + return not self._closed + + @property + def next_tick_time(self) -> datetime | None: + """The wall clock time when the next tick should happen, or `None` if it is not running.""" + return None if self._closed else self._next_tick_time + + def reset(self) -> None: + """Reset the timer to start timing from now (plus an optional alignment). + + If the timer was closed, or not started yet, it will be started. + """ + self._closed = False + self._update_next_tick_time() + self._current_tick_info = None + # We assume the clocks will behave similarly after the timer was reset, so we + # purposefully don't reset the clocks info. + _logger.debug("reset(): _next_tick_time=%s", self._next_tick_time) + + @override + def close(self) -> None: + """Close and stop the timer. + + Once `close` has been called, all subsequent calls to `ready()` will immediately + return False and calls to `consume()` / `receive()` or any use of the async + iterator interface will raise a + [`ReceiverStoppedError`][frequenz.channels.ReceiverStoppedError]. + + You can restart the timer with `reset()`. + """ + self._closed = True + # We need to make sure it's not None, otherwise `ready()` will start it + self._next_tick_time = datetime.now(timezone.utc) + + def _should_resync(self, info: ClocksInfo | timedelta | None) -> bool: + """Check if the timer needs to resynchronize with the wall clock. + + This checks if the wall clock jumped beyond the configured threshold, which + is defined in the timer configuration. + + Args: + info: The information about the clocks and their drift. If `None`, it will + not check for a resync, and will return `False`. If it is a + `ClocksInfo`, it will check the `wall_clock_jump` property. If it is a + `timedelta`, it will check if the absolute value is greater than the + configured threshold. + + Returns: + Whether the timer should resync to the wall clock. + """ + threshold = self._config.wall_clock_jump_threshold + if threshold is None or info is None: + return False + if isinstance(info, ClocksInfo): + info = info.wall_clock_jump + return abs(info) > threshold + + # We need to disable too many branches here, because the method is too complex but + # it is not trivial to split into smaller parts. + @override + async def ready(self) -> bool: # pylint: disable=too-many-branches + """Wait until the timer `interval` passed. + + Once a call to `ready()` has finished, the resulting tick information + must be read with a call to `consume()` (`receive()` or iterated over) + to tell the timer it should wait for the next interval. + + The timer will remain ready (this method will return immediately) + until it is consumed. + + Returns: + Whether the timer was started and it is still running. + """ + # If there are messages waiting to be consumed, return immediately. + if self._current_tick_info is not None: + return True + + # If `_next_tick_time` is `None`, it means it was created with + # `auto_start=True` and should be started. + if self._next_tick_time is None: + self.reset() + assert ( + self._next_tick_time is not None + ), "This should be assigned by reset()" + + # If a close was explicitly requested, we bail out. + if self._closed: + return False + + wall_clock_now = datetime.now(timezone.utc) + wall_clock_time_to_next_tick = self._next_tick_time - wall_clock_now + + # If we didn't reach the tick yet, sleep until we do. + # We need to do this in a loop to react to resets, time jumps and wall clock + # time compression, in which cases we need to recalculate the time to the next + # tick and try again. + sleeps: list[ClocksInfo] = [] + should_resync: bool = self._should_resync(self._clocks_info) + while wall_clock_time_to_next_tick > _TD_ZERO: + prev_clocks_info = self._clocks_info + # We don't assign directly to self._clocks_info because its type is + # ClocksInfo | None, and sleep() returns ClocksInfo, so we can avoid some + # None checks further in the code with `clocks_info` (and we make the code + # more succinct). + clocks_info = await self._sleep( + wall_clock_time_to_next_tick, prev_clocks_info=prev_clocks_info + ) + should_resync = self._should_resync(clocks_info) + wall_clock_now = datetime.now(timezone.utc) + self._clocks_info = clocks_info + + sleeps.append(clocks_info) + + if previous_factor := self._has_drifted_beyond_tolerance( + new_clocks_info=clocks_info, prev_clocks_info=prev_clocks_info + ): + # If we are resyncing we have a different issue, and we are not going to + # use the factor to adjust the clock, but will just resync + if not should_resync: + _logger.warning( + "The wall clock time drifted too much from the monotonic time. The " + "monotonic time will be adjusted to compensate for this difference. " + "We expected the wall clock time to have advanced (%s), but the " + "monotonic time advanced (%s) [previous_factor=%s " + "current_factor=%s, factor_change_absolute_tolerance=%s].", + clocks_info.wall_clock_elapsed, + clocks_info.monotonic_elapsed, + previous_factor, + clocks_info.wall_clock_factor, + self._config.wall_clock_drift_tolerance_factor, + ) + + wall_clock_time_to_next_tick = self._next_tick_time - wall_clock_now + + # Technically the monotonic drift should always be positive, but we handle + # negative values just in case, we've seen a lot of weird things happen. + monotonic_drift = abs(clocks_info.monotonic_drift) + drift_tolerance = self._config.async_drift_tolerance + if drift_tolerance is not None and monotonic_drift > drift_tolerance: + _logger.warning( + "The timer was supposed to sleep for %s, but it slept for %s " + "instead [difference=%s, tolerance=%s]. This is likely due to a " + "task taking too much time to complete and blocking the event " + "loop for too long. You probably should profile your code to " + "find out what's taking too long.", + clocks_info.monotonic_requested_sleep, + clocks_info.monotonic_elapsed, + monotonic_drift, + drift_tolerance, + ) + + # If we detect a time jump, we exit the loop and handle it outside of it, to + # also account for time jumps in the past that could happen without even + # having entered into the sleep loop. + if should_resync: + _logger.debug( + "ready(): Exiting the wait loop because we detected a time jump " + "and need to re-sync." + ) + break + + if _logger.isEnabledFor(logging.DEBUG): + _logger.debug( + "ready(): In sleep loop:\n" + " next_tick_time=%s (%s)\n" + " now=%s (%s)\n" + " mono_now=%s\n" + " wall_clock_time_to_next_tick=%s (%s)", + self._next_tick_time, + self._next_tick_time.timestamp(), + wall_clock_now, + wall_clock_now.timestamp(), + asyncio.get_running_loop().time(), + wall_clock_time_to_next_tick, + wall_clock_time_to_next_tick.total_seconds(), + ) + + # If there was a time jump, we need to resync the timer to the wall clock, + # otherwise we can be sleeping for a long time until the timer catches up, + # which is not suitable for many use cases. + # + # Resyncing the timer ensures that we keep ticking more or less at `interval` + # even in the event of time jumps, with the downside that the timer will + # trigger more than once for the same timestamp if it jumps back in time, + # and will skip ticks if it jumps forward in time. + # + # When there is no threshold, so there is no resync, the ticks will be + # contigous in time from the wall clock perspective, waiting until we reach + # the expected next tick time when jumping back in time, and bursting all + # missed ticks when jumping forward in time. + if should_resync: + assert self._clocks_info is not None + _logger.warning( + "The wall clock jumped %s (%s seconds) in time (threshold=%s). " + "A tick will be triggered immediately with the `expected_tick_time` " + "as it was before the time jump and the timer will be resynced to " + "the wall clock.", + self._clocks_info.wall_clock_jump, + self._clocks_info.wall_clock_jump.total_seconds(), + self._config.wall_clock_jump_threshold, + ) + + # If a close was explicitly requested during the sleep, we bail out. + if self._closed: + return False + + self._current_tick_info = TickInfo( + expected_tick_time=self._next_tick_time, sleep_infos=sleeps + ) + + if should_resync: + _logger.debug( + "ready(): Before resync:\n" + " next_tick_time=%s\n" + " now=%s\n" + " wall_clock_time_to_next_tick=%s", + self._next_tick_time, + wall_clock_now, + wall_clock_time_to_next_tick, + ) + self._update_next_tick_time(now=wall_clock_now) + _logger.debug( + "ready(): After resync: next_tick_time=%s", self._next_tick_time + ) + else: + self._next_tick_time += self._interval + _logger.debug( + "ready(): No resync needed: next_tick_time=%s", + self._next_tick_time, + ) + + return True + + @override + def consume(self) -> TickInfo: + """Return the latest tick information once `ready()` is complete. + + Once the timer has triggered + ([`ready()`][frequenz.sdk.timeseries.WallClockTimer.ready] is done), this method + returns the information about the tick that just happened. + + Returns: + The information about the tick that just happened. + + Raises: + ReceiverStoppedError: If the timer was closed via `close()`. + """ + # If it was closed and there it no pending result, we raise + # (if there is a pending result, then we still want to return it first) + if self._closed and self._current_tick_info is None: + raise ReceiverStoppedError(self) + + assert ( + self._current_tick_info is not None + ), "calls to `consume()` must be follow a call to `ready()`" + info = self._current_tick_info + self._current_tick_info = None + return info + + def _update_next_tick_time(self, *, now: datetime | None = None) -> None: + """Update the next tick time, aligning it to `self._align_to` or now.""" + if now is None: + now = datetime.now(timezone.utc) + + elapsed = _TD_ZERO + + if self._config.align_to is not None: + elapsed = (now - self._config.align_to) % self._interval + + self._next_tick_time = now + self._interval - elapsed + + def _has_drifted_beyond_tolerance( + self, *, new_clocks_info: ClocksInfo, prev_clocks_info: ClocksInfo | None + ) -> float | Literal[False]: + """Check if the wall clock drifted beyond the configured tolerance. + + This checks the relative difference between the wall clock and monotonic time + based on the `wall_clock_drift_tolerance_factor` configuration. + + Args: + new_clocks_info: The information about the clocks and their drift from the + current sleep. + prev_clocks_info: The information about the clocks and their drift from the + previous sleep. If `None`, the previous factor will be considered 1.0. + + Returns: + The previous wall clock factor if the drift is beyond the tolerance, or + `False` if it is not. + """ + tolerance = self._config.wall_clock_drift_tolerance_factor + if tolerance is None: + return False + + previous_factor = ( + prev_clocks_info.wall_clock_factor if prev_clocks_info else 1.0 + ) + current_factor = new_clocks_info.wall_clock_factor + if abs(current_factor - previous_factor) > tolerance: + return previous_factor + return False + + async def _sleep( + self, delay: timedelta, /, *, prev_clocks_info: ClocksInfo | None + ) -> ClocksInfo: + """Sleep for a given time and return information about the clocks and their drift. + + The time to sleep is adjusted based on the previously observed drift between the + wall clock and monotonic time, if any. + + Also saves the information about the clocks and their drift for the next sleep. + + Args: + delay: The time to sleep. Will be adjusted based on `prev_clocks_info` if + available. + prev_clocks_info: The information about the clocks and their drift from the + previous sleep. If `None`, the sleep will be done as requested, without + adjusting the time to sleep. + + Returns: + The information about the clocks and their drift for this sleep. + """ + if prev_clocks_info is not None: + _logger.debug( + "_sleep(): Adjusted original requested delay (%s) with factor %s", + delay.total_seconds(), + prev_clocks_info.wall_clock_factor, + ) + delay = prev_clocks_info.wall_clock_to_monotonic(delay) + + delay_s = delay.total_seconds() + + _logger.debug("_sleep(): Will sleep for %s seconds", delay_s) + start_monotonic_time = asyncio.get_running_loop().time() + start_wall_clock_time = datetime.now(timezone.utc) + await asyncio.sleep(delay_s) + + end_monotonic_time = asyncio.get_running_loop().time() + end_wall_clock_time = datetime.now(timezone.utc) + + elapsed_monotonic = timedelta(seconds=end_monotonic_time - start_monotonic_time) + elapsed_wall_clock = end_wall_clock_time - start_wall_clock_time + + wall_clock_jump = elapsed_wall_clock - elapsed_monotonic + should_resync = self._should_resync(wall_clock_jump) + _logger.debug("_sleep(): SHOULD RESYNC? %s", should_resync) + clocks_info = ClocksInfo( + monotonic_requested_sleep=delay, + monotonic_time=end_monotonic_time, + wall_clock_time=end_wall_clock_time, + monotonic_elapsed=elapsed_monotonic, + wall_clock_elapsed=elapsed_wall_clock, + # If we should resync it means there was a big time jump, which should be + # exceptional (NTP adjusting the clock or something like that), in this case + # we want to use the previous factor as the current one will be way off. + wall_clock_factor=( + prev_clocks_info.wall_clock_factor + if prev_clocks_info and should_resync + else float("nan") # nan means let ClocksInfo calculate it + ), + ) + _logger.debug( + "_sleep(): After sleeping:\n" + " monotonic_requested_sleep=%s\n" + " monotonic_time=%s\n" + " wall_clock_time=%s\n" + " monotonic_elapsed=%s\n" + " wall_clock_elapsed=%s\n" + " factor=%s\n", + clocks_info.monotonic_requested_sleep, + clocks_info.monotonic_time, + clocks_info.wall_clock_time, + clocks_info.monotonic_elapsed, + clocks_info.wall_clock_elapsed, + clocks_info.wall_clock_factor, + ) + + return clocks_info + + def __str__(self) -> str: + """Return a string representation of this timer.""" + return f"{type(self).__name__}({self.interval})" + + def __repr__(self) -> str: + """Return a string representation of this timer.""" + next_tick = ( + "" + if self._next_tick_time is None + else f", next_tick_time={self._next_tick_time!r}" + ) + return ( + f"{type(self).__name__}" + ) From dbee422557ee20f36b9d8d774c08d0cc8a6ee92c Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 09:28:42 +0200 Subject: [PATCH 03/32] Add `ClocksInfo` tests Signed-off-by: Leandro Lucarella --- .../wall_clock_timer/test_clocksinfo.py | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 tests/timeseries/_resampling/wall_clock_timer/test_clocksinfo.py diff --git a/tests/timeseries/_resampling/wall_clock_timer/test_clocksinfo.py b/tests/timeseries/_resampling/wall_clock_timer/test_clocksinfo.py new file mode 100644 index 000000000..55b7e30da --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/test_clocksinfo.py @@ -0,0 +1,253 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Tests for the `ClocksInfo` class.""" + +import math +import re +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +import pytest + +from frequenz.sdk.timeseries._resampling._wall_clock_timer import ClocksInfo + +_DEFAULT_MONOTONIC_REQUESTED_SLEEP = timedelta(seconds=1.0) +_DEFAULT_MONOTONIC_TIME = 1234.5 +_DEFAULT_WALL_CLOCK_TIME = datetime(2023, 1, 1, tzinfo=timezone.utc) +_DEFAULT_MONOTONIC_ELAPSED = timedelta(seconds=1.1) +_DEFAULT_WALL_CLOCK_ELAPSED = timedelta(seconds=1.2) + + +@pytest.mark.parametrize( + "elapsed", + [timedelta(seconds=0), timedelta(seconds=-1.0)], + ids=["zero", "negative"], +) +def test_monotonic_requested_sleep_invalid(elapsed: timedelta) -> None: + """Test monotonic_requested_sleep with invalid values.""" + with pytest.raises( + ValueError, + match=r"^monotonic_requested_sleep must be strictly positive, not " + + re.escape(repr(elapsed)) + + r"$", + ): + _ = ClocksInfo( + monotonic_requested_sleep=elapsed, + monotonic_time=_DEFAULT_MONOTONIC_TIME, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=_DEFAULT_MONOTONIC_ELAPSED, + wall_clock_elapsed=_DEFAULT_WALL_CLOCK_ELAPSED, + ) + + +@pytest.mark.parametrize( + "time", + [float("-inf"), float("nan"), float("inf")], +) +def test_monotonic_time_invalid(time: float) -> None: + """Test monotonic_time with invalid values.""" + with pytest.raises( + ValueError, + match=rf"^monotonic_time must be a number, not {re.escape(repr(time))}$", + ): + _ = ClocksInfo( + monotonic_requested_sleep=_DEFAULT_MONOTONIC_REQUESTED_SLEEP, + monotonic_time=time, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=_DEFAULT_MONOTONIC_ELAPSED, + wall_clock_elapsed=_DEFAULT_WALL_CLOCK_ELAPSED, + ) + + +@pytest.mark.parametrize( + "elapsed", + [timedelta(seconds=0), timedelta(seconds=-1.0)], + ids=["zero", "negative"], +) +def test_monotonic_elapsed_invalid(elapsed: timedelta) -> None: + """Test monotonic_elapsed with invalid values.""" + with pytest.raises( + ValueError, + match="^monotonic_elapsed must be strictly positive, not " + rf"{re.escape(repr(elapsed))}$", + ): + _ = ClocksInfo( + monotonic_requested_sleep=_DEFAULT_MONOTONIC_REQUESTED_SLEEP, + monotonic_time=_DEFAULT_MONOTONIC_TIME, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=elapsed, + wall_clock_elapsed=_DEFAULT_WALL_CLOCK_ELAPSED, + ) + + +@pytest.mark.parametrize("wall_clock_factor", [float("nan"), 2.3]) +def test_clocks_info_construction(wall_clock_factor: float) -> None: + """Test that ClocksInfo can be constructed and attributes are set correctly.""" + monotonic_requested_sleep = timedelta(seconds=1.0) + monotonic_time = 1234.5 + wall_clock_time = datetime(2023, 1, 1, tzinfo=timezone.utc) + monotonic_elapsed = timedelta(seconds=1.1) + wall_clock_elapsed = timedelta(seconds=1.2) + + info = ClocksInfo( + monotonic_requested_sleep=monotonic_requested_sleep, + monotonic_time=monotonic_time, + wall_clock_time=wall_clock_time, + monotonic_elapsed=monotonic_elapsed, + wall_clock_elapsed=wall_clock_elapsed, + wall_clock_factor=wall_clock_factor, + ) + + assert info.monotonic_requested_sleep == monotonic_requested_sleep + assert info.monotonic_time == monotonic_time + assert info.wall_clock_time == wall_clock_time + assert info.monotonic_elapsed == monotonic_elapsed + assert info.wall_clock_elapsed == wall_clock_elapsed + + # Check in particular that using nan explicitly is the same as using the default + # We test how the default is calculated in another test + if math.isnan(wall_clock_factor): + assert info == ClocksInfo( + monotonic_requested_sleep=monotonic_requested_sleep, + monotonic_time=monotonic_time, + wall_clock_time=wall_clock_time, + monotonic_elapsed=monotonic_elapsed, + wall_clock_elapsed=wall_clock_elapsed, + ) + else: + assert info.wall_clock_factor == wall_clock_factor + + +@pytest.mark.parametrize( + "requested_sleep, monotonic_elapsed, expected_drift", + [ + (timedelta(seconds=1.0), timedelta(seconds=1.1), timedelta(seconds=0.1)), + (timedelta(seconds=1.0), timedelta(seconds=0.9), timedelta(seconds=-0.1)), + (timedelta(seconds=1.0), timedelta(seconds=1.0), timedelta(seconds=0.0)), + ], + ids=["positive", "negative", "no_drift"], +) +def test_monotonic_drift( + requested_sleep: timedelta, + monotonic_elapsed: timedelta, + expected_drift: timedelta, +) -> None: + """Test the monotonic_drift property.""" + info = ClocksInfo( + monotonic_requested_sleep=requested_sleep, + monotonic_time=_DEFAULT_MONOTONIC_TIME, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=monotonic_elapsed, + wall_clock_elapsed=_DEFAULT_WALL_CLOCK_ELAPSED, + ) + assert info.monotonic_drift == pytest.approx(expected_drift) + + +@pytest.mark.parametrize( + "wall_clock_elapsed, monotonic_elapsed, expected_jump", + [ + (timedelta(seconds=1.0), timedelta(seconds=2.1), timedelta(seconds=-1.1)), + (timedelta(seconds=1.0), timedelta(seconds=0.19), timedelta(seconds=0.81)), + (timedelta(seconds=1.0), timedelta(seconds=1.0), timedelta(seconds=0.0)), + ], + ids=["positive", "negative", "no_jump"], +) +def test_wall_clock_jump( + wall_clock_elapsed: timedelta, + monotonic_elapsed: timedelta, + expected_jump: timedelta, +) -> None: + """Test the wall_clock_jump property.""" + info = ClocksInfo( + monotonic_requested_sleep=_DEFAULT_MONOTONIC_REQUESTED_SLEEP, + monotonic_time=_DEFAULT_MONOTONIC_TIME, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=monotonic_elapsed, + wall_clock_elapsed=wall_clock_elapsed, + ) + assert info.wall_clock_jump == pytest.approx(expected_jump) + + +@dataclass(kw_only=True, frozen=True) +class _TestCaseWallClockFactor: + """Test case for wall clock factor calculation.""" + + id: str + monotonic_elapsed: timedelta + wall_clock_elapsed: timedelta + expected_factor: float + + +@pytest.mark.parametrize( + "case", + [ + _TestCaseWallClockFactor( + id="wall_faster", + monotonic_elapsed=timedelta(seconds=1.0), + wall_clock_elapsed=timedelta(seconds=1.1), + expected_factor=0.9090909090909091, + ), + _TestCaseWallClockFactor( + id="wall_slower", + monotonic_elapsed=timedelta(seconds=1.0), + wall_clock_elapsed=timedelta(seconds=0.9), + expected_factor=1.11111111111111, + ), + _TestCaseWallClockFactor( + id="in_sync", + monotonic_elapsed=timedelta(seconds=1.0), + wall_clock_elapsed=timedelta(seconds=1.0), + expected_factor=1.0, + ), + _TestCaseWallClockFactor( + id="wall_twice_as_fast", + monotonic_elapsed=timedelta(seconds=0.5), + wall_clock_elapsed=timedelta(seconds=1.0), + expected_factor=0.5, + ), + ], + ids=lambda case: case.id, +) +def test_wall_clock_factor(case: _TestCaseWallClockFactor) -> None: + """Test the calculate_wall_clock_factor method with valid inputs.""" + info = ClocksInfo( + monotonic_requested_sleep=_DEFAULT_MONOTONIC_REQUESTED_SLEEP, + monotonic_time=_DEFAULT_MONOTONIC_TIME, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=case.monotonic_elapsed, + wall_clock_elapsed=case.wall_clock_elapsed, + ) + assert info.wall_clock_factor == pytest.approx(case.expected_factor) + assert info.wall_clock_to_monotonic(case.wall_clock_elapsed) == pytest.approx( + case.monotonic_elapsed + ) + + +@pytest.mark.parametrize( + "elapsed", + [timedelta(seconds=0), timedelta(seconds=-1.0)], + ids=["zero", "negative"], +) +def test_wall_clock_factor_invalid_wall_clock_elapsed( + elapsed: timedelta, caplog: pytest.LogCaptureFixture +) -> None: + """Test that a warning is logged when wall_clock_elapsed is zero.""" + expected_log = ( + "The monotonic clock advanced 0:00:01, but the wall clock " + f"stayed still or jumped back (elapsed: {elapsed})!" + ) + with caplog.at_level("WARNING"): + info = ClocksInfo( + monotonic_requested_sleep=_DEFAULT_MONOTONIC_REQUESTED_SLEEP, + monotonic_time=_DEFAULT_MONOTONIC_TIME, + wall_clock_time=_DEFAULT_WALL_CLOCK_TIME, + monotonic_elapsed=timedelta(seconds=1.0), + wall_clock_elapsed=elapsed, + ) + assert info.wall_clock_to_monotonic(timedelta(seconds=1.0)) == timedelta( + seconds=10.0 + ) + assert info.wall_clock_factor == 10.0 + + assert expected_log in caplog.text From e89d38d187f4e1234f28f25735ced5585ba73e77 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 10:07:09 +0200 Subject: [PATCH 04/32] Add tests for `WallClockTimerConfig` Signed-off-by: Leandro Lucarella --- .../wall_clock_timer/test_config.py | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tests/timeseries/_resampling/wall_clock_timer/test_config.py diff --git a/tests/timeseries/_resampling/wall_clock_timer/test_config.py b/tests/timeseries/_resampling/wall_clock_timer/test_config.py new file mode 100644 index 000000000..d38662c2b --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/test_config.py @@ -0,0 +1,187 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Tests for the `WallClockTimerConfig` class.""" + + +import math +import re +from datetime import datetime, timedelta, timezone + +import pytest +from frequenz.core.datetime import UNIX_EPOCH + +from frequenz.sdk.timeseries._resampling._wall_clock_timer import WallClockTimerConfig + + +def test_from_interval_defaults() -> None: + """Test WallClockTimerConfig.from_interval() with only interval (all defaults).""" + interval = timedelta(seconds=10) + config = WallClockTimerConfig.from_interval(interval) + assert config.align_to == UNIX_EPOCH + assert config.async_drift_tolerance == pytest.approx(timedelta(seconds=1.0)) + assert config.wall_clock_drift_tolerance_factor == pytest.approx(0.1) + assert config.wall_clock_jump_threshold == pytest.approx(timedelta(seconds=10.0)) + + +def test_from_interval_all_args() -> None: + """Test WallClockTimerConfig.from_interval() with all arguments provided.""" + interval = timedelta(seconds=5) + align_to = datetime(2023, 1, 1, tzinfo=timezone.utc) + async_factor = 0.2 + wall_factor = 0.3 + jump_factor = 0.4 + config = WallClockTimerConfig.from_interval( + interval, + align_to=align_to, + async_drift_tolerance_factor=async_factor, + wall_clock_drift_tolerance_factor=wall_factor, + wall_clock_jump_threshold_factor=jump_factor, + ) + assert config.align_to == align_to + assert config.async_drift_tolerance == pytest.approx(timedelta(seconds=1.0)) + assert config.wall_clock_drift_tolerance_factor == pytest.approx(0.3) + assert config.wall_clock_jump_threshold == pytest.approx(timedelta(seconds=2.0)) + + +@pytest.mark.parametrize( + "interval", [timedelta(seconds=0), timedelta(seconds=-1)], ids=str +) +def test_from_interval_invalid(interval: timedelta) -> None: + """Test WallClockTimerConfig.from_interval() with invalid interval raises ValueError.""" + with pytest.raises(ValueError, match=r"^interval must be bigger than 0, not "): + WallClockTimerConfig.from_interval(interval) + + +def test_trivial_defaults() -> None: + """Test that WallClockTimerConfig can be constructed with all defaults.""" + config = WallClockTimerConfig() + assert config.align_to == UNIX_EPOCH + assert config.async_drift_tolerance is None + assert config.wall_clock_drift_tolerance_factor is None + assert config.wall_clock_jump_threshold is None + + +def test_all_valid_arguments() -> None: + """Test that WallClockTimerConfig can be constructed with all valid arguments.""" + align_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + async_drift_tolerance = timedelta(seconds=5) + wall_clock_drift_tolerance_factor = 0.5 + wall_clock_jump_threshold = timedelta(seconds=10) + config = WallClockTimerConfig( + align_to=align_to, + async_drift_tolerance=async_drift_tolerance, + wall_clock_drift_tolerance_factor=wall_clock_drift_tolerance_factor, + wall_clock_jump_threshold=wall_clock_jump_threshold, + ) + assert config.align_to == align_to + assert config.async_drift_tolerance == async_drift_tolerance + assert config.wall_clock_drift_tolerance_factor == wall_clock_drift_tolerance_factor + assert config.wall_clock_jump_threshold == wall_clock_jump_threshold + + +@pytest.mark.parametrize( + "align_to", + [None, datetime(2020, 1, 1, tzinfo=timezone.utc)], + ids=str, +) +def test_valid_align_to(align_to: datetime | None) -> None: + """Test that align_to is accepted and set for valid input.""" + config = WallClockTimerConfig(align_to=align_to) + assert config.align_to == align_to + + +def test_align_to_timezone_unaware() -> None: + """Test checks on the resampling buffer.""" + with pytest.raises( + ValueError, match=r"^align_to (.*) should be a timezone aware datetime$" + ): + _ = WallClockTimerConfig(align_to=datetime(2020, 1, 1, tzinfo=None)) + + +_VALID_NUMBERS = [ + 0.1, + 1.0, + None, +] +_VALID_TIMEDELTAS = [ + *(timedelta(seconds=v) for v in _VALID_NUMBERS if v is not None), + None, +] + +_INVALID_NUMBERS = [ + -0.0001, + 0.0, + -1, + 0, + float("inf"), + float("-inf"), + float("nan"), +] + +_INVALID_TIMEDELTAS = [ + *(timedelta(seconds=v) for v in _INVALID_NUMBERS if math.isfinite(v)), +] + + +@pytest.mark.parametrize("value", _VALID_TIMEDELTAS, ids=str) +def test_valid_async_drift_tolerance(value: timedelta | None) -> None: + """Test that async_drift_tolerance accepts valid values.""" + config = WallClockTimerConfig(async_drift_tolerance=value) + assert config.async_drift_tolerance == value + + +@pytest.mark.parametrize("value", _INVALID_TIMEDELTAS, ids=str) +def test_invalid_async_drift_tolerance(value: timedelta | None) -> None: + """Test that strictly positive fields reject invalid values (matrix).""" + with pytest.raises( + ValueError, + match=rf"^async_drift_tolerance should be positive or None, not {re.escape(repr(value))}$", + ): + _ = WallClockTimerConfig(async_drift_tolerance=value) + + +@pytest.mark.parametrize("value", _VALID_TIMEDELTAS, ids=str) +def test_valid_wall_clock_jump_threshold( + value: timedelta | None, +) -> None: + """Test that wall_clock_jump_threshold accepts valid values.""" + config = WallClockTimerConfig(wall_clock_jump_threshold=value) + assert config.wall_clock_jump_threshold == value + + +@pytest.mark.parametrize("value", _INVALID_TIMEDELTAS, ids=str) +def test_invalid_wall_clock_jump_threshold( + value: timedelta | None, +) -> None: + """Test that strictly positive fields reject invalid values (matrix).""" + with pytest.raises( + ValueError, + match=r"^wall_clock_jump_threshold should be positive or None, not " + + re.escape(repr(value)) + + r"$", + ): + _ = WallClockTimerConfig(wall_clock_jump_threshold=value) + + +@pytest.mark.parametrize("value", [0.1, 1.0, 1, None]) +def test_valid_wall_clock_drift_tolerance_factor( + value: float | None, +) -> None: + """Test that strictly positive fields accept valid values.""" + config = WallClockTimerConfig(wall_clock_drift_tolerance_factor=value) + assert config.wall_clock_drift_tolerance_factor == value + + +@pytest.mark.parametrize("value", _INVALID_NUMBERS, ids=str) +def test_invalid_wall_clock_drift_tolerance_factor( + value: float | None, +) -> None: + """Test that strictly positive fields reject invalid values (matrix).""" + with pytest.raises( + ValueError, + match=r"^wall_clock_drift_tolerance_factor should be positive or None, not " + + re.escape(repr(value)) + + r"$", + ): + _ = WallClockTimerConfig(wall_clock_drift_tolerance_factor=value) From 0088d2e776a9fe690dc81378439adc726a362928 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 10:58:10 +0200 Subject: [PATCH 05/32] Add `conftest.py` for the wall clock timer tests For now, we include one fixture that we will use to mock the `datetime` class imported in the wall clock timer definition, so we can control the current time in the tests (and set the default to the UNIX epoch to make it easier to read times in the tests). We don't use time-machine for 2 reasons: 1. We will also need to use async-solipsism and time-machine also mocks the monotonic timer, sometimes conflicting with async-solipsism. 2. We want to tell only when sleep was called within the wall clock timer module, not in the tests for example. Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/conftest.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/timeseries/_resampling/wall_clock_timer/conftest.py diff --git a/tests/timeseries/_resampling/wall_clock_timer/conftest.py b/tests/timeseries/_resampling/wall_clock_timer/conftest.py new file mode 100644 index 000000000..6d7278f46 --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/conftest.py @@ -0,0 +1,21 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Fixtures for wall clock timer tests.""" + +from collections.abc import Iterator +from datetime import datetime +from unittest.mock import MagicMock, patch + +import pytest +from frequenz.core.datetime import UNIX_EPOCH + + +@pytest.fixture +def datetime_mock() -> Iterator[MagicMock]: + """Mock the datetime class in the target module and set now to the UNIX epoch.""" + dt_symbol = "frequenz.sdk.timeseries._resampling._wall_clock_timer.datetime" + dt_mock = MagicMock(name="datetime_mock", wraps=datetime, spec_set=datetime) + dt_mock.now.return_value = UNIX_EPOCH + with patch(dt_symbol, new=dt_mock): + yield dt_mock From 1bbf467223405f38c615a1cc9d1de1b97ece58ee Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 13:20:56 +0200 Subject: [PATCH 06/32] Add utility functions for testing the wall clock timer This commit adds a utility module for testing the wall clock timer. The utility functions include a method to convert various time-related types (datetime, timedelta, float) to seconds, and a function to get the current wall clock time from the mocked datetime (when the `datetime_mock` fixture is used). Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/util.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/timeseries/_resampling/wall_clock_timer/util.py diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py new file mode 100644 index 000000000..558406d21 --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -0,0 +1,44 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Utility functions for testing the wall clock timer.""" + +from datetime import datetime, timedelta, timezone +from typing import assert_never, overload + + +@overload +def to_seconds(value: datetime | timedelta | float) -> float: ... + + +@overload +def to_seconds(value: None) -> None: ... + + +def to_seconds(value: datetime | timedelta | float | None) -> float | None: + """Convert a datetime, timedelta, or float to seconds.""" + match value: + case datetime(): + return value.timestamp() + case timedelta(): + return value.total_seconds() + case float() | int() | None: + return value + case unexpected: + assert_never(unexpected) + + +# This is needed to work with the datetime_mock fixture in conftest.py +def wall_now() -> datetime: + """Get the current wall clock time from the mocked datetime in the target module.""" + # Disable isort formatting because it wants to put the ignore in the wrong line + # We now also have to ignore the maximum line length (E501) + # isort: off + # pylint: disable-next=import-outside-toplevel + from frequenz.sdk.timeseries._resampling._wall_clock_timer import ( # type: ignore[attr-defined] # noqa: E501 + datetime as mock_datetime, + ) + + # isort: on + + return mock_datetime.now(timezone.utc) From 2a1fc6ea693fc591ca331bc40445dde512639265 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 13:22:24 +0200 Subject: [PATCH 07/32] Add __init__.py files to resampling tests We need this to be able to import then new `util.py` module in the tests. Signed-off-by: Leandro Lucarella --- tests/timeseries/_resampling/__init__.py | 4 ++++ tests/timeseries/_resampling/wall_clock_timer/__init__.py | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 tests/timeseries/_resampling/__init__.py create mode 100644 tests/timeseries/_resampling/wall_clock_timer/__init__.py diff --git a/tests/timeseries/_resampling/__init__.py b/tests/timeseries/_resampling/__init__.py new file mode 100644 index 000000000..14f32ce6f --- /dev/null +++ b/tests/timeseries/_resampling/__init__.py @@ -0,0 +1,4 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Resampling tests.""" diff --git a/tests/timeseries/_resampling/wall_clock_timer/__init__.py b/tests/timeseries/_resampling/wall_clock_timer/__init__.py new file mode 100644 index 000000000..b85cb059c --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/__init__.py @@ -0,0 +1,4 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Wall clock timer tests.""" From b43b31afdc7e37344c7f2937142bb17509275386 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 13:24:08 +0200 Subject: [PATCH 08/32] Add basic `WallClockTimer` tests These tests only test the basic functionality, like construction, string representation, etc. The ticking behaviour test will be done in a sepate file in a follow-up commit. Signed-off-by: Leandro Lucarella --- .../wall_clock_timer/test_timer_basic.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/timeseries/_resampling/wall_clock_timer/test_timer_basic.py diff --git a/tests/timeseries/_resampling/wall_clock_timer/test_timer_basic.py b/tests/timeseries/_resampling/wall_clock_timer/test_timer_basic.py new file mode 100644 index 000000000..c7638c457 --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/test_timer_basic.py @@ -0,0 +1,103 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Basic tests for `WallClockTimer`.""" + +import re +from datetime import timedelta +from unittest.mock import MagicMock + +import pytest + +from frequenz.sdk.timeseries._resampling._wall_clock_timer import ( + WallClockTimer, + WallClockTimerConfig, +) + +from .util import to_seconds, wall_now + +pytestmark = pytest.mark.usefixtures("datetime_mock") + + +@pytest.mark.parametrize( + "interval", + [timedelta(seconds=0.0), timedelta(seconds=-0.01)], + ids=["zero", "negative"], +) +def test_invalid_interval(interval: timedelta) -> None: + """Test WallClockTimer with invalid intervals raises ValueError.""" + with pytest.raises( + ValueError, + match=rf"^interval must be positive, not {re.escape(str(interval))}$", + ): + _ = WallClockTimer(interval) + + +def test_custom_config() -> None: + """Test WallClockTimer with a custom configuration.""" + interval = timedelta(seconds=5) + config = MagicMock(name="config", spec=WallClockTimerConfig) + + timer = WallClockTimer(interval, config=config, auto_start=False) + assert timer.interval == interval + assert timer.config is config + + +def test_auto_start_default() -> None: + """Test WallClockTimer uses auto_start=True by default.""" + interval = timedelta(seconds=1) + timer = WallClockTimer(interval) + assert timer.interval == interval + assert timer.config == WallClockTimerConfig.from_interval(interval) + assert timer.is_running + assert timer.next_tick_time == wall_now() + interval + + +def test_auto_start_disabled() -> None: + """Test WallClockTimer does not start when auto_start=False.""" + interval = timedelta(seconds=1) + timer = WallClockTimer(interval, auto_start=False) + assert timer.interval == interval + assert timer.config == WallClockTimerConfig.from_interval(interval) + assert not timer.is_running + assert timer.next_tick_time is None + + +def test_reset() -> None: + """Test WallClockTimer.reset() starts the timer and sets next_tick_time relative to now.""" + interval = timedelta(seconds=3) + timer = WallClockTimer(interval, auto_start=False) + timer.reset() + assert timer.is_running + assert timer.next_tick_time is not None + assert to_seconds(timer.next_tick_time) == pytest.approx( + to_seconds(wall_now() + interval) + ) + + +def test_close() -> None: + """Test WallClockTimer.close() stops the timer and returns no next_tick_time.""" + interval = timedelta(seconds=2) + timer = WallClockTimer(interval, auto_start=True) + timer.close() + assert not timer.is_running + assert timer.next_tick_time is None + + +def test_str() -> None: + """Test __str__ returns only the interval.""" + interval = timedelta(seconds=4) + timer = WallClockTimer(interval, auto_start=False) + assert str(timer) == f"WallClockTimer({interval})" + + +def test_repr() -> None: + """Test __repr__ includes interval and running state.""" + interval = timedelta(seconds=4) + timer = WallClockTimer(interval, auto_start=False) + assert repr(timer) == f"WallClockTimer" + timer.reset() + assert ( + repr(timer) == f"WallClockTimer" + ) From 41a595d8a511f20bb058c27e3269c4a723f6dd03 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 15:19:05 +0200 Subject: [PATCH 09/32] Add a pytest tool to compare time approximately This tool can compare datetime or timedelta objects approximately, like pytest.approx(). It uses an absolute 1ms tolerance by default. Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/util.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py index 558406d21..7e222c982 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/util.py +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -6,6 +6,14 @@ from datetime import datetime, timedelta, timezone from typing import assert_never, overload +# This is not great, we are depending on an internal pytest API, but it is +# the most convenient way to provide a custom approx() comparison for datetime +# and timedelta. +# Other alternatives proven to be even more complex and hacky. +# It also looks like we are not the only ones doing this, see: +# https://github.com/pytest-dev/pytest/issues/8395 +from _pytest.python_api import ApproxBase + @overload def to_seconds(value: datetime | timedelta | float) -> float: ... @@ -42,3 +50,48 @@ def wall_now() -> datetime: # isort: on return mock_datetime.now(timezone.utc) + + +# Pylint complains about abstract-method because _yield_comparisons is not implemented +# but it is used only in the default __eq__ method, which we are re-defining, so we can +# ignore it. +class approx_time(ApproxBase): # pylint: disable=invalid-name, abstract-method + """Perform approximate comparisons for datetime or timedelta objects. + + Inherits from `ApproxBase` to provide a rich comparison output in pytest. + """ + + expected: datetime | timedelta + abs: timedelta + + def __init__( + self, + expected: datetime | timedelta, + *, + abs: timedelta = timedelta(milliseconds=1), # pylint: disable=redefined-builtin + ) -> None: + """Initialize this instance.""" + if abs < timedelta(): + raise ValueError( + f"absolute tolerance must be a non-negative timedelta, not {abs}" + ) + super().__init__(expected, abs=abs) + + def __repr__(self) -> str: + """Return a string representation of this instance.""" + return f"{self.expected} ± {self.abs}" + + def __eq__(self, actual: object) -> bool: + """Compare this instance with another object.""" + # We need to split the cases for datetime and timedelta for type checking + # reasons. + diff: timedelta + match (self.expected, actual): + case (datetime(), datetime()): + diff = self.expected - actual + case (timedelta(), timedelta()): + diff = self.expected - actual + case _: + return NotImplemented + + return abs(diff) <= self.abs From 720e04001d88436afddfaf936fb84a8914de62d8 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 15:22:07 +0200 Subject: [PATCH 10/32] Add a utility to compare `TickInfo` objects approximately Like pytest.approx(), but compares `TickInfo` objects. It uses an absolute 1ms tolerance by default. Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/util.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py index 7e222c982..ff046c22a 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/util.py +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -6,6 +6,8 @@ from datetime import datetime, timedelta, timezone from typing import assert_never, overload +import pytest + # This is not great, we are depending on an internal pytest API, but it is # the most convenient way to provide a custom approx() comparison for datetime # and timedelta. @@ -14,6 +16,8 @@ # https://github.com/pytest-dev/pytest/issues/8395 from _pytest.python_api import ApproxBase +from frequenz.sdk.timeseries._resampling._wall_clock_timer import ClocksInfo, TickInfo + @overload def to_seconds(value: datetime | timedelta | float) -> float: ... @@ -95,3 +99,92 @@ def __eq__(self, actual: object) -> bool: return NotImplemented return abs(diff) <= self.abs + + +# We need to rewrite most of the attributes in these classes to use approximate +# comparisons. This means if a new field is added, we need to update the +# approx_tick_info function below to handle the new fields. To catch this, we do +# a sanity check here, so if new fields are added we get an early warning instead of +# getting tests errors because of some rounding error in a newly added field. +assert set(ClocksInfo.__dataclass_fields__.keys()) == { + "monotonic_requested_sleep", + "monotonic_time", + "wall_clock_time", + "monotonic_elapsed", + "wall_clock_elapsed", + "wall_clock_factor", +}, "ClocksInfo fields were added or removed, please update the approx_tick_info function." +assert set(TickInfo.__dataclass_fields__.keys()) == { + "expected_tick_time", + "sleep_infos", +}, "TickInfo fields were added or removed, please update the approx_tick_info function." + + +def approx_tick_info( + expected: TickInfo, + *, + abs: timedelta = timedelta(milliseconds=1), # pylint: disable=redefined-builtin +) -> TickInfo: + """Create a copy of a `TickInfo` object with approximate comparisons. + + Fields are replaced by approximate comparison objects (`approx_time` or + `pytest.approx`). + + This version bypasses `__post_init__` to avoid validation `TypeError`s. + + Args: + expected: The expected `TickInfo` object to compare against. + abs: The absolute tolerance as a `timedelta` for all time-based + comparisons. Defaults to 1ms. + + Returns: + A new `TickInfo` instance ready for approximate comparison. + """ + abs_s = abs.total_seconds() + approx_sleeps = [] + for s_info in expected.sleep_infos: + # HACK: Create a blank instance to bypass __init__ and __post_init__. + # This prevents the TypeError from the validation logic. + approx_s = object.__new__(ClocksInfo) + + # Use object.__setattr__ to assign fields to the frozen instance. + object.__setattr__( + approx_s, + "monotonic_requested_sleep", + approx_time(s_info.monotonic_requested_sleep, abs=abs), + ) + # Use the standard pytest.approx for float values + object.__setattr__( + approx_s, "monotonic_time", pytest.approx(s_info.monotonic_time, abs=abs_s) + ) + object.__setattr__( + approx_s, "wall_clock_time", approx_time(s_info.wall_clock_time, abs=abs) + ) + object.__setattr__( + approx_s, + "monotonic_elapsed", + approx_time(s_info.monotonic_elapsed, abs=abs), + ) + object.__setattr__( + approx_s, + "wall_clock_elapsed", + approx_time(s_info.wall_clock_elapsed, abs=abs), + ) + if s_info.wall_clock_factor is not None: + object.__setattr__( + approx_s, + "wall_clock_factor", + pytest.approx(s_info.wall_clock_factor, abs=abs_s), + ) + approx_sleeps.append(approx_s) + + # Do the same for the top-level frozen TickInfo object + approx_tick_info = object.__new__(TickInfo) + object.__setattr__( + approx_tick_info, + "expected_tick_time", + approx_time(expected.expected_tick_time, abs=abs), + ) + object.__setattr__(approx_tick_info, "sleep_infos", approx_sleeps) + + return approx_tick_info From 1f336c2609fa4363874e577dc97c8008ca1341e0 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 15:23:43 +0200 Subject: [PATCH 11/32] Add an utility to assert that a string matches a regex pattern Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/util.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py index ff046c22a..2290a1a1e 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/util.py +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -3,6 +3,7 @@ """Utility functions for testing the wall clock timer.""" +import re from datetime import datetime, timedelta, timezone from typing import assert_never, overload @@ -15,6 +16,7 @@ # It also looks like we are not the only ones doing this, see: # https://github.com/pytest-dev/pytest/issues/8395 from _pytest.python_api import ApproxBase +from typing_extensions import override from frequenz.sdk.timeseries._resampling._wall_clock_timer import ClocksInfo, TickInfo @@ -188,3 +190,21 @@ def approx_tick_info( object.__setattr__(approx_tick_info, "sleep_infos", approx_sleeps) return approx_tick_info + + +class matches_re: # pylint: disable=invalid-name + """Assert that a given string (or string representation) matches a regex pattern.""" + + def __init__(self, pattern: str, flags: int = 0) -> None: + """Initialize with a regex pattern and optional flags.""" + self._regex = re.compile(pattern, flags) + + @override + def __eq__(self, other: object) -> bool: + """Check if the string representation of `other` matches the regex pattern.""" + return bool(self._regex.match(str(other))) + + @override + def __repr__(self) -> str: + """Return a string representation of this instance.""" + return self._regex.pattern From 9d9efe03ab5c867b0ffdf9b8df8fbba01ea38215 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Wed, 30 Jul 2025 15:30:57 +0200 Subject: [PATCH 12/32] Add a fixture to monitor `asyncio.sleep()` calls We monitor only the calls to sleep inside the wall clock time module, so we can assert those calls independently from what happens in the tests themselves, for example. Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/conftest.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/timeseries/_resampling/wall_clock_timer/conftest.py b/tests/timeseries/_resampling/wall_clock_timer/conftest.py index 6d7278f46..954f38265 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/conftest.py +++ b/tests/timeseries/_resampling/wall_clock_timer/conftest.py @@ -3,9 +3,10 @@ """Fixtures for wall clock timer tests.""" +import asyncio from collections.abc import Iterator from datetime import datetime -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from frequenz.core.datetime import UNIX_EPOCH @@ -19,3 +20,16 @@ def datetime_mock() -> Iterator[MagicMock]: dt_mock.now.return_value = UNIX_EPOCH with patch(dt_symbol, new=dt_mock): yield dt_mock + + +@pytest.fixture +def asyncio_sleep_mock() -> Iterator[AsyncMock]: + """Mock asyncio.sleep in the target module for all tests.""" + asyncio_symbol = "frequenz.sdk.timeseries._resampling._wall_clock_timer.asyncio" + mock = AsyncMock( + name="asyncio_sleep_mock", wraps=asyncio.sleep, spec_set=asyncio.sleep + ) + asyncio_mock = AsyncMock(name="asyncio_mock", wraps=asyncio, spec_set=asyncio) + asyncio_mock.sleep = mock + with patch(asyncio_symbol, new=asyncio_mock): + yield mock From d12254b946f0601250f224303a6d7a6423d0112b Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 14:57:52 +0200 Subject: [PATCH 13/32] Add utilities to create datetime/timedelta from seconds Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/util.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py index 2290a1a1e..e32946abc 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/util.py +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -42,6 +42,20 @@ def to_seconds(value: datetime | timedelta | float | None) -> float | None: assert_never(unexpected) +def timestamp(ts: datetime | float, /) -> datetime: + """Convert a timestamp in seconds since the epoch to a UTC datetime.""" + if isinstance(ts, datetime): + return ts + return datetime.fromtimestamp(ts, tz=timezone.utc) + + +def delta(sec: float | timedelta, /) -> timedelta: + """Create a timedelta from seconds.""" + if isinstance(sec, timedelta): + return sec + return timedelta(seconds=sec) + + # This is needed to work with the datetime_mock fixture in conftest.py def wall_now() -> datetime: """Get the current wall clock time from the mocked datetime in the target module.""" From 8ad7aa78cd4ce932ffd137a0ec65dd2f2df43911 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 14:58:34 +0200 Subject: [PATCH 14/32] Add shortcut to get the monotonic (loop) time Signed-off-by: Leandro Lucarella --- tests/timeseries/_resampling/wall_clock_timer/util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py index e32946abc..6b5fed104 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/util.py +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -72,6 +72,11 @@ def wall_now() -> datetime: return mock_datetime.now(timezone.utc) +def mono_now() -> float: + """Get the current monotonic time.""" + return asyncio.get_running_loop().time() + + # Pylint complains about abstract-method because _yield_comparisons is not implemented # but it is used only in the default __eq__ method, which we are re-defining, so we can # ignore it. From 5c4c18312bbd1e63a35addbd672899ef2a211bce Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 15:14:49 +0200 Subject: [PATCH 15/32] Add utility to mock the clocks for complex ticking tests The `TimeDriver` class encapsulates the necessary mocks for `datetime.datetime` and provides methods to manipulate wall clock time during tests. This is particularly useful for testing components that rely on both wall clock time (which can be adjusted by the system) and monotonic time (which should always move forward). We also register the util module so pytest rewrite asserts, as it uses asserts to verify some stuff behaves as expected, and it is nicer to get those asserts as test failures instead of programming errors. Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/conftest.py | 5 + .../_resampling/wall_clock_timer/util.py | 185 +++++++++++++++++- 2 files changed, 188 insertions(+), 2 deletions(-) diff --git a/tests/timeseries/_resampling/wall_clock_timer/conftest.py b/tests/timeseries/_resampling/wall_clock_timer/conftest.py index 954f38265..4d8168e88 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/conftest.py +++ b/tests/timeseries/_resampling/wall_clock_timer/conftest.py @@ -12,6 +12,11 @@ from frequenz.core.datetime import UNIX_EPOCH +# Some of the utils do assertions and we want them to be rewritten by pytest for better +# error messages +pytest.register_assert_rewrite("tests.timeseries._resampling.wall_clock_timer.util") + + @pytest.fixture def datetime_mock() -> Iterator[MagicMock]: """Mock the datetime class in the target module and set now to the UNIX epoch.""" diff --git a/tests/timeseries/_resampling/wall_clock_timer/util.py b/tests/timeseries/_resampling/wall_clock_timer/util.py index 6b5fed104..1068c467e 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/util.py +++ b/tests/timeseries/_resampling/wall_clock_timer/util.py @@ -3,9 +3,14 @@ """Utility functions for testing the wall clock timer.""" +import asyncio +import logging import re +from collections.abc import Coroutine, Sequence +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone -from typing import assert_never, overload +from typing import NamedTuple, TypeVar, assert_never, overload +from unittest.mock import MagicMock import pytest @@ -18,7 +23,11 @@ from _pytest.python_api import ApproxBase from typing_extensions import override -from frequenz.sdk.timeseries._resampling._wall_clock_timer import ClocksInfo, TickInfo +from frequenz.sdk.timeseries import ClocksInfo, TickInfo + +_logger = logging.getLogger(__name__) + +_T = TypeVar("_T") @overload @@ -227,3 +236,175 @@ def __eq__(self, other: object) -> bool: def __repr__(self) -> str: """Return a string representation of this instance.""" return self._regex.pattern + + +class Adjustment(NamedTuple): + """A time adjustment to be applied at a specific monotonic time.""" + + mono_delta: timedelta | float + """The monotonic time delta to sleep before adjusting the wall clock time, in seconds.""" + + wall_time: datetime | float + """The new wall clock time to set at that monotonic time, in seconds since the epoch in UTC.""" + + +@dataclass(kw_only=True, frozen=True) +class TimeDriver: + """A utility for driving the wall and monotonic clocks in tests. + + This class encapsulates the necessary mocks for `datetime.datetime` and + provides methods to manipulate wall clock time during tests. This is particularly + useful for testing components that rely on both wall clock time (which can be + adjusted by the system) and monotonic time (which should always move forward). + + It is designed to be used as a pytest fixture, where `datetime_mock` is + provided by another fixture. + + The main method to use in tests is `next_tick()`, which simulates time + passing and wall clock adjustments while waiting for a timer tick. + """ + + datetime_mock: MagicMock + """A mock for the `datetime` module.""" + + loop: asyncio.AbstractEventLoop = field(default_factory=asyncio.get_event_loop) + """The asyncio event loop.""" + + mono_start: float = field(default_factory=mono_now) + """The starting monotonic time of the test.""" + + wall_start: datetime = field(default_factory=wall_now) + """The starting wall clock time of the test.""" + + def __post_init__(self) -> None: + """Initialize the time driver by logging the start times.""" + _logger.debug( + "Start: wall_now=%s, mono_now=%s", self.wall_start, self.mono_start + ) + + async def _shift_time( + self, + wall_delta: timedelta | float, + *, + mono_delta: timedelta | float | None = None, + ) -> tuple[datetime, float]: + """Advance the time by the given number of seconds. + + This advances both the wall clock and the time machine fake time. + + Args: + wall_delta: The amount of time to advance the wall clock, in seconds or as a + timedelta. + mono_delta: The amount of time to advance the monotonic clock, in seconds or + as a timedelta. If None, it defaults to the same value as `wall_time`. + + Returns: + A tuple containing the new wall clock time and the new monotonic time. + """ + wall_delta = to_seconds(wall_delta) + mono_delta = to_seconds(mono_delta) + if mono_delta is None: + mono_delta = wall_delta + _logger.debug( + "_shift_time(): wall_delta=%s, mono_delta=%s", wall_delta, mono_delta + ) + + wall_start = wall_now() + mono_start = mono_now() + + _logger.debug( + "_shift_time(): Before sleep: wall_now=%s, mono_now=%s", + wall_start, + mono_start, + ) + await asyncio.sleep(mono_delta) + self.datetime_mock.now.return_value = wall_now() + timedelta(seconds=wall_delta) + _logger.debug( + "_shift_time(): After shift: wall_now=%s, mono_now=%s", + wall_now(), + mono_now(), + ) + + new_wall_now = wall_now() + new_mono_now = mono_now() + _logger.debug( + "NEW TIME: new_wall_now=%s, new_mono_now=%s", new_wall_now, new_mono_now + ) + + assert new_wall_now == approx_time(wall_start + delta(wall_delta)) + assert new_mono_now == pytest.approx(mono_start + mono_delta) + + return new_wall_now, new_mono_now + + def _update_wall_clock(self, new_time: datetime | float) -> None: + """Update the wall clock time to the given datetime.""" + new_time = timestamp(new_time) + _logger.debug( + "_update_wall_clock(): at mono_now=%s %s -> %s (%s -> %s)", + mono_now(), + to_seconds(wall_now()), + to_seconds(new_time), + wall_now(), + new_time, + ) + self.datetime_mock.now.return_value = new_time + _logger.debug( + "_update_wall_clock(): wall clock updated to %s (%s)", + to_seconds(wall_now()), + wall_now(), + ) + + def _shift_wall_clock(self, shift_delta: timedelta | float) -> None: + """Shift the wall clock by the given timedelta.""" + shift_delta = delta(shift_delta) + new_wall_now = wall_now() + shift_delta + self._update_wall_clock(new_wall_now) + + async def next_tick( + self, + next_tick_wall_times: Sequence[Adjustment], + coro: Coroutine[None, None, _T], + ) -> _T: + """Wait for the next tick of the timer and return the result of the coroutine. + + This method simulates the passage of time and wall clock adjustments while a + timer is waiting for its next tick. It runs a provided coroutine (like a + timer's `receive()` or `ready()` method) and, while it's running, applies a + series of time adjustments. + + This is useful for simulating wall clock jumps or drifts. The adjustments are + applied after the timer has started waiting, ensuring the timer correctly + observes the time change. + + Args: + next_tick_wall_times: A sequence of `Adjustment` tuples. Each tuple + specifies a monotonic time delta to wait before setting the wall + clock to a new time. This simulates wall clock adjustments + happening while the timer is sleeping. + coro: The coroutine to run that will receive the next tick from the + timer. Typically this will be the timer's `receive()` or `ready()` + method. + + Returns: + The result of the `coro` coroutine. + """ + async with asyncio.TaskGroup() as tg: + _logger.debug("_next_tick(): Creating timer task for receive()") + timer_task: asyncio.Task[_T] = tg.create_task(coro) + for adj in next_tick_wall_times: + sleep_time = to_seconds(adj.mono_delta) + wall_time = to_seconds(adj.wall_time) + _logger.debug( + "_next_tick(): Waiting for %s seconds before setting wall clock to %s", + sleep_time, + wall_time, + ) + await asyncio.sleep(sleep_time) + assert not timer_task.done() + self._update_wall_clock(adj.wall_time) + _logger.debug( + "_next_tick(): After setting wall clock: now_wall=%s, now_mono=%s", + to_seconds(wall_now()), + mono_now(), + ) + return await timer_task From a9e91d5b4535595e599e0e04f5ae9c54fd583096 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 15:21:36 +0200 Subject: [PATCH 16/32] Disable `pylint` `wrong-import-position` check This check is also performed by `flake8`. Signed-off-by: Leandro Lucarella --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 40ba4cedd..0aeac0439 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,7 @@ disable = [ "unnecessary-lambda-assignment", "unused-import", "unused-variable", + "wrong-import-position", ] [tool.pylint.design] From f381d34248e105c072c6a6689314b40513ff57b9 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 15:21:58 +0200 Subject: [PATCH 17/32] Add a fixture to easily use the `TimeDriver` Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/conftest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/timeseries/_resampling/wall_clock_timer/conftest.py b/tests/timeseries/_resampling/wall_clock_timer/conftest.py index 4d8168e88..29698a745 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/conftest.py +++ b/tests/timeseries/_resampling/wall_clock_timer/conftest.py @@ -16,6 +16,9 @@ # error messages pytest.register_assert_rewrite("tests.timeseries._resampling.wall_clock_timer.util") +# We need to import this module after registering the assert rewrite +from .util import TimeDriver # noqa: E402 + @pytest.fixture def datetime_mock() -> Iterator[MagicMock]: @@ -38,3 +41,11 @@ def asyncio_sleep_mock() -> Iterator[AsyncMock]: asyncio_mock.sleep = mock with patch(asyncio_symbol, new=asyncio_mock): yield mock + + +@pytest.fixture +async def time_driver(datetime_mock: MagicMock) -> TimeDriver: + """Fixture to mock the clocks environment for testing.""" + return TimeDriver( + datetime_mock=datetime_mock, + ) From 2e5f9a9ab26bba9d699b5c4b54118c965091225a Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 15:25:10 +0200 Subject: [PATCH 18/32] Add ticking behavior tests These tests cover the behavior of the WallClockTimer when it is ticking, including how it handles wall clock and monotonic clock synchronization, drift, and jumps. The tests include scenarios where the clocks are in sync, where there is a constant drift (both forward and backward), and where there are jumps in the wall clock time. The tests also verify that the timer adjusts its sleep time accordingly and that it logs appropriate warnings when the wall clock time drifts too much from the monotonic time. These tests cover for the most common cases, but are not comprehensive. With the current test infrastructure, it should be possible to add tests for more scenarios fairly easily in the future as needed. Signed-off-by: Leandro Lucarella --- .../wall_clock_timer/test_timer_ticking.py | 743 ++++++++++++++++++ 1 file changed, 743 insertions(+) create mode 100644 tests/timeseries/_resampling/wall_clock_timer/test_timer_ticking.py diff --git a/tests/timeseries/_resampling/wall_clock_timer/test_timer_ticking.py b/tests/timeseries/_resampling/wall_clock_timer/test_timer_ticking.py new file mode 100644 index 000000000..4f128aece --- /dev/null +++ b/tests/timeseries/_resampling/wall_clock_timer/test_timer_ticking.py @@ -0,0 +1,743 @@ +# License: MIT +# Copyright © 2025 Frequenz Energy-as-a-Service GmbH + +"""Tests for the ticking behavior of the WallClockTimer. + +This module contains tests to verify that the `WallClockTimer` behaves correctly +under various clock conditions, such as clock drift and time jumps. + +The core of the testing is done in the `test_ticking` function, which is a +parameterized test that runs multiple scenarios defined as `_TickTestCase` +instances. Each test case simulates a sequence of timer ticks, specifying how the +wall clock should be manipulated during each tick's sleep phase. + +Key aspects of the testing approach: + +- **Time Mocking**: We mock both the wall clock (`datetime.now`) and monotonic + clock (`asyncio.sleep`) to have full control over time. The `TimeDriver` + utility class orchestrates the time advancements and adjustments. + +- **Simplified Time Representation**: All time values in the test cases (timestamps, + deltas) are expressed in plain seconds (as floats). This is possible because + the mocked wall clock starts at the Unix epoch (time 0). This convention + greatly simplifies writing and debugging tests, as it's easier to reason + about small numbers rather than complex `datetime` and `timedelta` objects, + which can be especially cryptic when they are large or negative. + +- **Scenario-based Testing**: Each `_TickTestCase` defines a complete scenario, + like "constant forward drift" or "backward jump". For each tick within the + scenario, a `_TickSpec` defines the expected `TickInfo` and any wall clock + adjustments to be made. This allows for precise testing of the timer's logic + for drift compensation and resynchronization. +""" + +import logging +import re +from collections.abc import Sequence +from unittest.mock import AsyncMock, call + +import async_solipsism +import pytest +from attr import dataclass + +from frequenz.sdk.timeseries._resampling._wall_clock_timer import ( + ClocksInfo, + TickInfo, + WallClockTimer, + WallClockTimerConfig, +) + +from .util import ( + Adjustment, + TimeDriver, + approx_tick_info, + approx_time, + delta, + matches_re, + mono_now, + timestamp, + to_seconds, + wall_now, +) + +_logger = logging.getLogger(__name__) + + +@pytest.fixture +def event_loop_policy() -> async_solipsism.EventLoopPolicy: + """Return an event loop policy that uses the async solipsism event loop.""" + return async_solipsism.EventLoopPolicy() + + +async def test_ready_called_twice_returns_immediately(time_driver: TimeDriver) -> None: + """Test that calling ready() twice returns immediately the second time.""" + interval = delta(1.0) + timer = WallClockTimer( + interval, config=WallClockTimerConfig.from_interval(interval) + ) + + # The first call to ready() will start the timer and wait for the first tick. + mono_start = mono_now() + await time_driver.next_tick([Adjustment(0.9, 1.0)], timer.ready()) + mono_end = mono_now() + assert mono_end == pytest.approx(mono_start + 1.0) + + # The second call should return immediately. + mono_start = mono_now() + assert await timer.ready() + mono_end = mono_now() + + # Time should not have advanced. + assert mono_end == pytest.approx(mono_start) + + +@dataclass(kw_only=True, frozen=True) +class _TickSpec: + """Specification for one tick in a TickTestCase.""" + + wall_clock_adjustments: Sequence[Adjustment] + """The wall clock time adjustments to do while waiting for the next tick.""" + + expected_tick_info: TickInfo + """Expected TickInfo returned by the timer after the tick, if any.""" + + expected_warnings: Sequence[str] = () + """The expected warning messages during the tick. + + Each string will be used as a regex pattern that should match the logged warning + messages. + """ + + +@dataclass(kw_only=True, frozen=True) +class _TickTestCase: + """A test case for the WallClockTimer ticking behavior.""" + + id: str + """The identifier for the test case, used in parameterized tests.""" + + ticks: Sequence[_TickSpec] + """The specifications for the ticks in this test case.""" + + +# IMPORTANT: All tests are written for a timer with a 1 second interval and default +# config. If the default changes, the tests will need to be updated accordingly. +@pytest.mark.parametrize( + "case", + [ + # Both clocks are perfecly in sync. The timer should sleep for the full + # interval and wake up at the expected wall clock time. + _TickTestCase( + id="in_sync", + ticks=[ + # We adjust the wall clock time slightly before the timer wakes up + # to the interval (1.0 seconds) to match the elapsed time in the + # monotonic clock, to ensure both clocks are in sync. + _TickSpec( # Tick 1, 2, 3, 4, 5 + wall_clock_adjustments=[Adjustment(0.9, 1.0 * (i + 1))], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0 * (i + 1)), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.0 * (i + 1)), + monotonic_time=1.0 * (i + 1), + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ) + for i in range(5) + ], + ), + # The wall clock is slightly ahead of the monotonic clock, but with a + # constant drift. The first tick will be slightly off because of the drift, + # the next tick the timer will adjust using a factor due to the drift, but + # also account for the drift in the first tick, ending up with a + # particularly short sleep time. Afterwards the timer should apply a factor + # and always sleep slightly less than the interval to account for the drift. + _TickTestCase( + id="constant_forward_drift_within_tolerance", + ticks=[ + # We adjust the wall clock time slightly before the timer wakes up + # to a bit more than the interval (1.01 seconds) to simulate a wall + # clock forward drift. + _TickSpec( # Tick 1 + wall_clock_adjustments=[Adjustment(0.9, 1.01)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.01), + monotonic_time=1.0, + wall_clock_elapsed=delta(1.01), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ), + # In the second tick, we expect the timer to detect that drift and + # adjust the sleep time accordingly, so it will sleep slightly less + # than the interval to account for the drift. It calculates an + # adjustment factor of 0.99 seconds but for this tick sleeps even + # less, because it also woke up late, so it effectively sleeps + # 0.980198 seconds. + # We adjust the wall clock time slightly before the timer wakes up + # to a perfect multiple of the interval (2.0 seconds) to simulate + # that the wall clock drift stays constant (so after the adjustments + # the timer could wake up at the desired wall clock time of 2 + # intervals). + _TickSpec( # Tick 2 + wall_clock_adjustments=[Adjustment(0.9, 2.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(2.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(2.0), + monotonic_time=1.980198, + wall_clock_elapsed=delta(0.99), + monotonic_elapsed=delta(0.980198), + monotonic_requested_sleep=delta(0.980198), + ) + ], + ), + ), + # For the rest of the ticks, we just expect the timer to always + # sleep for the interval * the factor (0.99099 seconds) to account + # for the drift, so we adjust the wall clock time to match the + # expected wall clock time for each tick at a multiple of the + # interval (3.0 seconds, 4.0 seconds, etc.) and verify the timer + # reports a constant factor (sleeps always for 0.990099 seconds). + *[ + _TickSpec( # Tick 3, 4, ..., 10 + wall_clock_adjustments=[Adjustment(0.9, 3.0 + i)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(3.0 + i), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(3.0 + i), + monotonic_time=2.970297 + i * 0.990099, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(0.990099), + monotonic_requested_sleep=delta(0.990099), + ) + ], + ), + ) + for i in range(8) + ], + ], + ), + # The wall clock is slightly behind the monotonic clock, but with a + # constant drift. The first tick will be slightly off because of the lag, + # the next tick the timer will adjust using a factor due to the lag, but + # also account for the lag in the first tick, ending up with a + # particularly long sleep time. Afterwards the timer should apply a factor + # and always sleep slightly more than the interval to account for the lag. + _TickTestCase( + id="constant_backward_drift_within_tolerance", + ticks=[ + # We adjust the wall clock time slightly before the timer wakes up + # to a bit less than the interval (0.99 seconds) to simulate a wall + # clock backward drift. + _TickSpec( # Tick 1 + wall_clock_adjustments=[ + Adjustment(0.9, 0.99), + Adjustment(0.105, 1.0), + ], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(0.99), + monotonic_time=1.0, + wall_clock_elapsed=delta(0.99), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ), + ClocksInfo( + wall_clock_time=timestamp(1.0), + monotonic_time=1.010101, + wall_clock_elapsed=delta(0.01), + monotonic_elapsed=delta(0.010101), + monotonic_requested_sleep=delta(0.010101), + ), + ], + ), + ), + _TickSpec( # Tick 2 + wall_clock_adjustments=[Adjustment(0.9, 2.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(2.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(2.0), + monotonic_time=2.020202, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.010101), + monotonic_requested_sleep=delta(1.010101), + ) + ], + ), + ), + *[ + _TickSpec( # Tick 3, 4, ..., 10 + wall_clock_adjustments=[Adjustment(0.9, 3.0 + i)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(3.0 + i), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(3.0 + i), + monotonic_time=3.030303 + i * 1.010101, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.010101), + monotonic_requested_sleep=delta(1.010101), + ) + ], + ), + ) + for i in range(8) + ], + ], + ), + # The wall clock is erratic and drifts forward and backwards but always within + # the tolerance drift tolerance. + _TickTestCase( + id="erratic_drift_without_jumps", + ticks=[ + # We adjust the wall clock time slightly before the timer wakes up + # to a bit more than the interval (1.01 seconds) to simulate a wall + # clock forward drift. + _TickSpec( # Tick 1 + wall_clock_adjustments=[Adjustment(0.9, 1.01)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.01), + monotonic_time=1.0, + wall_clock_elapsed=delta(1.01), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ), + # The timer should have adjusted for the forward drift. Now we + # introduce a backward drift. + _TickSpec( # Tick 2 + wall_clock_adjustments=[ + Adjustment(0.9, 1.99), + # So the timer will need to sleep again to compensate + Adjustment(0.09, 2.07), + ], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(2.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.99), + monotonic_time=1.980198, + wall_clock_elapsed=delta(0.98), + monotonic_elapsed=delta(0.980198), + monotonic_requested_sleep=delta(0.980198), + ), + ClocksInfo( + wall_clock_time=timestamp(2.07), + monotonic_time=1.9902, + wall_clock_elapsed=delta(0.08), + monotonic_elapsed=delta(0.010002), + monotonic_requested_sleep=delta(0.010002), + ), + ], + ), + # We get a warning this time because the last time was WAY off, + # it wanted to sleep for 0.01002 seconds but the wall clock + # advanced 0.080000 seconds, so the factor changed too much, + # which should trigger a warning. + # The total difference between the clocks is still way under the + # jump tolerance, so we don't get a resync. + expected_warnings=[ + re.escape( + "The wall clock time drifted too much from the monotonic time. The " + "monotonic time will be adjusted to compensate for this " + "difference. We expected the wall clock time to have advanced " + "(0:00:00.080000), but the monotonic time advanced " + "(0:00:00.010002) [previous_factor=1.00020204" + ) + + r"\d*" + + re.escape( + " current_factor=0.125025, factor_change_absolute_tolerance=0.1]." + ), + ], + ), + # The timer should have adjusted for the backward drift. Now we + # introduce a cumulative backward drift. + # The clock adjustment factor is now way off because of the last + # small sleep that took too long, so the timer will compensate for + # it, and sleep 0.116273 seconds when trying to sleep for 0.93 + # seconds (3.0 - 2.07). + _TickSpec( # Tick 3 + wall_clock_adjustments=[ + # Start monotonic_time=1.9902 + Adjustment(0.11, 2.998), + # monotonic_time=2.1002 (next sleep end at 2.106473) + Adjustment(0.0064, 2.999), + # monotonic_time=2.1066 (next sleep end at 2.106724) + Adjustment(0.0002, 3.0), + # monotonic_time=2.1068 (next sleep end at 2.106975) + ], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(3.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(2.998), + monotonic_time=2.106473, + wall_clock_elapsed=delta(0.928), + monotonic_elapsed=delta(0.116273), + monotonic_requested_sleep=delta(0.116273), + ), + ClocksInfo( + wall_clock_time=timestamp(2.999), + monotonic_time=2.106724, + wall_clock_elapsed=delta(0.001), + monotonic_elapsed=delta(0.000251), + monotonic_requested_sleep=delta(0.000251), + ), + ClocksInfo( + wall_clock_time=timestamp(3.0), + monotonic_time=2.106975, + wall_clock_elapsed=delta(0.001000), + monotonic_elapsed=delta(0.000251), + monotonic_requested_sleep=delta(0.000251), + ), + ], + ), + # Again there is a big difference in the drifts of different sleeps + expected_warnings=[ + re.escape( + "The wall clock time drifted too much from the monotonic time. The " + "monotonic time will be adjusted to compensate for this " + "difference. We expected the wall clock time to have advanced " + "(0:00:00.001000), but the monotonic time advanced " + "(0:00:00.000251) [previous_factor=0.125294" + ) + + r"\d*" + + re.escape( + " current_factor=0.251, factor_change_absolute_tolerance=0.1]." + ), + ], + ), + # Now it goes back to a forward drift + _TickSpec( # Tick 4 + wall_clock_adjustments=[Adjustment(0.2, 4.01)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(4.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(4.01), + monotonic_time=2.3579749, + wall_clock_elapsed=delta(1.01), + monotonic_elapsed=delta(0.251), + monotonic_requested_sleep=delta(0.251), + ) + ], + ), + ), + # And finally it stabilizes to a monotonic clock that is much + # faster than the wall clock, so the sleeps are constant but much smaller + # than the interval to make up for the drift. + _TickSpec( # Tick 5 + wall_clock_adjustments=[Adjustment(0.2, 5.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(5.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(5.0), + monotonic_time=2.604005, + wall_clock_elapsed=delta(0.99), + monotonic_elapsed=delta(0.246030), + monotonic_requested_sleep=delta(0.246030), + ) + ], + ), + ), + *[ + _TickSpec( # Tick 6, 7, ..., 10 + wall_clock_adjustments=[Adjustment(0.2, 6.0 + i)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(6.0 + i), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(6.0 + i), + monotonic_time=2.852519 + i * 0.248515, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(0.248515), + monotonic_requested_sleep=delta(0.248515), + ) + ], + ), + ) + for i in range(5) + ], + ], + ), + _TickTestCase( + id="forward_jump", + ticks=[ + # First tick is in sync + _TickSpec( # Tick 1 + wall_clock_adjustments=[Adjustment(0.9, 1.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.0), + monotonic_time=1.0, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ), + # The second tick has a forward jump of 1.01 seconds, which is more + # than the jump tolerance of 1 second, so we expect a warning and a + # resync of the timer to the wall clock. The clocks info will also + # have an override for the wall clock factor to use the previous + # one, because the factor will be either extremely low otherwise due + # to the jump. + _TickSpec( # Tick 2 + wall_clock_adjustments=[Adjustment(0.9, 3.01)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(2.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(3.01), + monotonic_time=2.0, + wall_clock_elapsed=delta(2.01), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + wall_clock_factor=1.0, + ) + ], + ), + expected_warnings=[ + re.escape( + "The wall clock jumped 0:00:01.010000 (1.01 seconds) in time " + "(threshold=0:00:01). A tick will be triggered immediately with " + "the `expected_tick_time` as it was before the time jump and the " + "timer will be resynced to the wall clock." + ), + ], + ), + _TickSpec( # Tick 3 + wall_clock_adjustments=[Adjustment(0.9, 4.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(4.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(4.0), + monotonic_time=2.99, + wall_clock_elapsed=delta(0.99), + monotonic_elapsed=delta(0.99), + monotonic_requested_sleep=delta(0.99), + ) + ], + ), + ), + # For the rest of the ticks, we just expect the clocks to keep being + # in sync, so the timer will sleep for the full interval + # (1.0 seconds) and wake up at the expected wall clock time. + *[ + _TickSpec( # Tick 4, 5, ..., 10 + wall_clock_adjustments=[Adjustment(0.9, 5.0 + i)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(5.0 + i), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(5.0 + i), + monotonic_time=3.99 + i, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ) + for i in range(7) + ], + ], + ), + _TickTestCase( + id="backward_jump", + ticks=[ + # First tick is in sync + _TickSpec( # Tick 1 + wall_clock_adjustments=[Adjustment(0.9, 1.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.0), + monotonic_time=1.0, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ), + # The second tick has a backward jump of 1.1 seconds, which is more + # than the jump tolerance of 1 second, so we expect a warning and a + # resync of the timer to the wall clock. + _TickSpec( # Tick 2 + wall_clock_adjustments=[Adjustment(0.9, 0.9)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(2.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(0.9), + monotonic_time=2.0, + wall_clock_elapsed=delta(-0.1), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + wall_clock_factor=1.0, + ) + ], + ), + expected_warnings=[ + re.escape( + "The wall clock jumped -1 day, 23:59:58.900000 (-1.1 seconds) in " + "time (threshold=0:00:01). A tick will be triggered immediately " + "with the `expected_tick_time` as it was before the time jump " + "and the timer will be resynced to the wall clock.", + ), + ], + ), + # After the jump, the timer is resynced. The next tick is set to be + # at 1.0, which is the next interval aligned to the epoch after 0.9 + # seconds. We make both clocks advance in sync for this tick, so + # after this all should get back to normal. + _TickSpec( # Tick 3 + wall_clock_adjustments=[Adjustment(0.09, 1.0)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(1.0), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(1.0), + monotonic_time=2.1, + wall_clock_elapsed=delta(0.1), + monotonic_elapsed=delta(0.1), + monotonic_requested_sleep=delta(0.1), + ) + ], + ), + ), + # For the rest of the ticks, we just expect the clocks to keep being + # in sync, so the timer will sleep for the full interval + # (1.0 seconds) and wake up at the expected wall clock time. + *[ + _TickSpec( # Tick 4, 5, ..., 10 + wall_clock_adjustments=[Adjustment(0.9, 2.0 + i)], + expected_tick_info=TickInfo( + expected_tick_time=timestamp(2.0 + i), + sleep_infos=[ + ClocksInfo( + wall_clock_time=timestamp(2.0 + i), + monotonic_time=3.1 + i, + wall_clock_elapsed=delta(1.0), + monotonic_elapsed=delta(1.0), + monotonic_requested_sleep=delta(1.0), + ) + ], + ), + ) + for i in range(7) + ], + ], + ), + ], + ids=lambda case: case.id, +) +async def test_ticking( + case: _TickTestCase, + time_driver: TimeDriver, + caplog: pytest.LogCaptureFixture, + asyncio_sleep_mock: AsyncMock, +) -> None: + """Test ticking behavior of the wall clock timer.""" + # You might need to comment this out if you want to enable debug logging + caplog.set_level( + logging.WARNING, + logger="frequenz.sdk.timeseries._resampling._wall_clock_timer", + ) + + # IMPORTANT: All test cases are relying on a timer with a 1 second interval and + # default config. If the default changes, the tests will need to be updated + # accordingly. + interval = delta(1.0) + timer = WallClockTimer( + interval, config=WallClockTimerConfig.from_interval(interval) + ) + + for i, tick in enumerate(case.ticks): + _logger.debug( + "================== %s: tick %s/%s =============================", + case.id, + i + 1, + len(case.ticks), + ) + tick = case.ticks[i] + expected_tick_info = tick.expected_tick_info + + mono_start = mono_now() + actual_tick_info = await time_driver.next_tick( + tick.wall_clock_adjustments, timer.receive() + ) + mono_end = mono_now() + _logger.debug( + "After tick %s: now_wall=%s, now_mono=%s, factor=%s", + i, + wall_now(), + mono_end, + ( + actual_tick_info.latest_sleep_info.wall_clock_factor + if actual_tick_info.latest_sleep_info + else None + ), + ) + + assert actual_tick_info == approx_tick_info(expected_tick_info) + + if actual_tick_info.latest_sleep_info: + assert actual_tick_info.latest_sleep_info.wall_clock_time == approx_time( + wall_now() + ) + assert actual_tick_info.sleep_infos[-1].monotonic_time == pytest.approx( + mono_end + ) + + if tick.wall_clock_adjustments: + assert timestamp(tick.wall_clock_adjustments[-1].wall_time) == approx_time( + wall_now() + ) + assert sum( + to_seconds(i.monotonic_elapsed) for i in actual_tick_info.sleep_infos + ) == pytest.approx(mono_end - mono_start) + assert asyncio_sleep_mock.mock_calls == [ + call(pytest.approx(to_seconds(t.monotonic_elapsed))) + for t in expected_tick_info.sleep_infos + ] + + filtered_logs = [ + r.message + for r in caplog.records + if r.levelno == logging.WARNING + and r.name == "frequenz.sdk.timeseries._resampling._wall_clock_timer" + ] + assert filtered_logs == [matches_re(w) for w in tick.expected_warnings] + + asyncio_sleep_mock.reset_mock() + caplog.clear() From 413c23e00d8eeddb9f1ab0bc383055277cdcb7e8 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Thu, 31 Jul 2025 15:28:07 +0200 Subject: [PATCH 19/32] Add custom comparison for `TickInfo` objects for pytest Testing the ticking behavior is very complicated in itself, and when there are failures, it is very difficult to understand what is happening and where the differences between the expected and actual tick times are. To help with this, we add a custom comparison for `TickInfo` objects that provides a detailed report of the differences between the expected and actual tick times, including the sleep information for each sleep. This uses the special `pytest` hook `pytest_assertrepr_compare`: https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_assertrepr_compare Signed-off-by: Leandro Lucarella --- .../_resampling/wall_clock_timer/conftest.py | 110 +++++++++++++++++- 1 file changed, 108 insertions(+), 2 deletions(-) diff --git a/tests/timeseries/_resampling/wall_clock_timer/conftest.py b/tests/timeseries/_resampling/wall_clock_timer/conftest.py index 29698a745..bb1eacaf8 100644 --- a/tests/timeseries/_resampling/wall_clock_timer/conftest.py +++ b/tests/timeseries/_resampling/wall_clock_timer/conftest.py @@ -4,13 +4,14 @@ """Fixtures for wall clock timer tests.""" import asyncio -from collections.abc import Iterator -from datetime import datetime +from collections.abc import Callable, Iterator, Sequence +from datetime import datetime, timedelta from unittest.mock import AsyncMock, MagicMock, patch import pytest from frequenz.core.datetime import UNIX_EPOCH +from frequenz.sdk.timeseries._resampling._wall_clock_timer import ClocksInfo, TickInfo # Some of the utils do assertions and we want them to be rewritten by pytest for better # error messages @@ -49,3 +50,108 @@ async def time_driver(datetime_mock: MagicMock) -> TimeDriver: return TimeDriver( datetime_mock=datetime_mock, ) + + +def pytest_assertrepr_compare(op: str, left: object, right: object) -> list[str] | None: + """Provide custom, readable error reports for TickInfo comparisons.""" + # We only care about == comparisons involving our TickInfo objects, returning None + # makes pytest fall back to its default comparison behavior. + if op != "==" or not isinstance(left, TickInfo) or not isinstance(right, TickInfo): + return None + + # Helper function to format values for readability + def format_val(val: object) -> str: + # For our time-based types, use str() for readability instead of repr() + if isinstance(val, (datetime, timedelta)): + return str(val) + # For our approx objects and others, the default repr is already good. + return repr(val) + + errors = _compare_tick_info_objects(left, right, format_val) + # If the comparison was actually successful (no errors), let pytest handle it + if not errors: + return None + + # Format the final error message + report = ["Comparing TickInfo objects:"] + report.append(" Differing attributes:") + report.append(f" {list(errors.keys())!r}") + report.append("") + + for field, diff in errors.items(): + report.append(f" Drill down into differing attribute '{field}':") + # The diff can be a simple tuple of (left, right) values or a list of + # strings for nested diffs + match diff: + case list(): + report.extend(f" {line}" for line in diff) + case (left_val, right_val): + report.append(f" - {format_val(left_val)}") + report.append(f" + {format_val(right_val)}") + case _: + assert False, f"Unexpected diff type: {type(diff)}" + + return report + + +# We need to compare the fields in TickInfo in a particular way in +# _compare_tick_info_objects. If new fields are added to the dataclass, we'll most +# likely need to add a custom comparison for those fields too. To catch this, we do +# a sanity check here, so if new fields are added we get an early warning instead of +# getting a comparison that misses those fields. +assert set(TickInfo.__dataclass_fields__.keys()) == { + "expected_tick_time", + "sleep_infos", +}, "TickInfo fields were added or removed, please update the _compare_tick_info_objects function." + + +def _compare_tick_info_objects( + left: TickInfo, right: TickInfo, format_val: Callable[[object], str] +) -> dict[str, object]: + """Compare two TickInfo objects and return a dictionary of differences.""" + errors: dict[str, object] = {} + + # 1. Compare top-level fields + if left.expected_tick_time != right.expected_tick_time: + errors["expected_tick_time"] = ( + left.expected_tick_time, + right.expected_tick_time, + ) + + # 2. Compare the list of ClocksInfo objects + sleeps_diff = _compare_sleep_infos_list( + left.sleep_infos, right.sleep_infos, format_val + ) + if sleeps_diff: + errors["sleep_infos"] = sleeps_diff + + return errors + + +def _compare_sleep_infos_list( + left: Sequence[ClocksInfo], + right: Sequence[ClocksInfo], + format_val: Callable[[object], str], +) -> list[str]: + """Compare two lists of ClocksInfo objects and return a list of error strings.""" + if len(left) != len(right): + return [ + f"List lengths differ: {len(left)} != {len(right)}", + f" {left!r}", + " !=", + f" {right!r}", + ] + + diffs: list[str] = [] + for i, (l_clock, r_clock) in enumerate(zip(left, right)): + if l_clock != r_clock: + diffs.append(f"Item at index [{i}] differs:") + # Get detailed diffs for the fields inside the ClocksInfo object + for field in l_clock.__dataclass_fields__: + l_val = getattr(l_clock, field) + r_val = getattr(r_clock, field) + if l_val != r_val: + diffs.append(f" Attribute '{field}':") + diffs.append(f" - {format_val(l_val)}") + diffs.append(f" + {format_val(r_val)}") + return diffs From bdce8adc9470066e3e86b734243d1339760ef816 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 11:09:13 +0200 Subject: [PATCH 20/32] Expose more resampling symbols publicly We expose defaults and the `Sink` and `Source` types because they are exposed to users. Signed-off-by: Leandro Lucarella --- src/frequenz/sdk/timeseries/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/frequenz/sdk/timeseries/__init__.py b/src/frequenz/sdk/timeseries/__init__.py index c57bbaa5e..cccfaaaed 100644 --- a/src/frequenz/sdk/timeseries/__init__.py +++ b/src/frequenz/sdk/timeseries/__init__.py @@ -40,8 +40,14 @@ from ._fuse import Fuse from ._moving_window import MovingWindow from ._periodic_feature_extractor import PeriodicFeatureExtractor -from ._resampling._base_types import SourceProperties -from ._resampling._config import ResamplerConfig, ResamplingFunction +from ._resampling._base_types import Sink, Source, SourceProperties +from ._resampling._config import ( + DEFAULT_BUFFER_LEN_INIT, + DEFAULT_BUFFER_LEN_MAX, + DEFAULT_BUFFER_LEN_WARN, + ResamplerConfig, + ResamplingFunction, +) from ._resampling._exceptions import ResamplingError, SourceStoppedError from ._resampling._wall_clock_timer import ( ClocksInfo, @@ -53,6 +59,9 @@ __all__ = [ "Bounds", "ClocksInfo", + "DEFAULT_BUFFER_LEN_INIT", + "DEFAULT_BUFFER_LEN_MAX", + "DEFAULT_BUFFER_LEN_WARN", "Fuse", "MovingWindow", "PeriodicFeatureExtractor", @@ -62,6 +71,8 @@ "ResamplingFunction", "Sample", "Sample3Phase", + "Sink", + "Source", "SourceProperties", "SourceStoppedError", "TickInfo", From ae942e07705538ac0597548d49cd81d2d06307e8 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 15:13:24 +0200 Subject: [PATCH 21/32] Import `ResamplerConfig` from `timeseries` instead of `actor` The proper location should be `timeseries` as one might need to use this class not only when using the actor. Signed-off-by: Leandro Lucarella --- docs/tutorials/getting_started.md | 4 ++-- examples/battery_pool.py | 2 +- src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py | 2 +- tests/microgrid/fixtures.py | 2 +- tests/timeseries/_battery_pool/test_battery_pool.py | 3 +-- .../_battery_pool/test_battery_pool_control_methods.py | 2 +- tests/timeseries/_pv_pool/test_pv_pool_control_methods.py | 2 +- tests/timeseries/mock_microgrid.py | 2 +- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 8c8bfbba2..d8620c8c1 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -32,7 +32,7 @@ import asyncio from datetime import timedelta from frequenz.sdk import microgrid -from frequenz.sdk.actor import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig ``` ## Create the application skeleton @@ -100,7 +100,7 @@ import asyncio from datetime import timedelta from frequenz.sdk import microgrid -from frequenz.sdk.actor import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig async def run() -> None: # This points to the default Frequenz microgrid sandbox diff --git a/examples/battery_pool.py b/examples/battery_pool.py index 79f543609..91070cb4c 100644 --- a/examples/battery_pool.py +++ b/examples/battery_pool.py @@ -11,7 +11,7 @@ from frequenz.channels import merge from frequenz.sdk import microgrid -from frequenz.sdk.actor import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig MICROGRID_API_URL = "grpc://microgrid.sandbox.api.frequenz.io:62060" diff --git a/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py b/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py index 1efe3056a..b7b20f059 100644 --- a/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py +++ b/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py @@ -33,7 +33,7 @@ class LogicalMeter: from datetime import timedelta from frequenz.sdk import microgrid - from frequenz.sdk.actor import ResamplerConfig + from frequenz.sdk.timeseries import ResamplerConfig from frequenz.client.microgrid import ComponentMetricId diff --git a/tests/microgrid/fixtures.py b/tests/microgrid/fixtures.py index 4985a8f7a..3d0ac0e6b 100644 --- a/tests/microgrid/fixtures.py +++ b/tests/microgrid/fixtures.py @@ -16,9 +16,9 @@ from pytest_mock import MockerFixture from frequenz.sdk import microgrid -from frequenz.sdk.actor import ResamplerConfig from frequenz.sdk.microgrid._power_distributing import ComponentPoolStatus from frequenz.sdk.microgrid.component_graph import _MicrogridComponentGraph +from frequenz.sdk.timeseries import ResamplerConfig from ..timeseries.mock_microgrid import MockMicrogrid from ..utils.component_data_streamer import MockComponentDataStreamer diff --git a/tests/timeseries/_battery_pool/test_battery_pool.py b/tests/timeseries/_battery_pool/test_battery_pool.py index 2da4f0585..fa4b6910d 100644 --- a/tests/timeseries/_battery_pool/test_battery_pool.py +++ b/tests/timeseries/_battery_pool/test_battery_pool.py @@ -30,12 +30,11 @@ MAX_BATTERY_DATA_AGE_SEC, WAIT_FOR_COMPONENT_DATA_SEC, ) -from frequenz.sdk.actor import ResamplerConfig from frequenz.sdk.microgrid._power_distributing import ComponentPoolStatus from frequenz.sdk.microgrid._power_distributing._component_managers._battery_manager import ( _get_battery_inverter_mappings, ) -from frequenz.sdk.timeseries import Bounds, Sample +from frequenz.sdk.timeseries import Bounds, ResamplerConfig, Sample from frequenz.sdk.timeseries._base_types import SystemBounds from frequenz.sdk.timeseries.battery_pool import BatteryPool from frequenz.sdk.timeseries.formula_engine._formula_generators._formula_generator import ( diff --git a/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py b/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py index cd21796f7..da54139ad 100644 --- a/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py +++ b/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py @@ -16,12 +16,12 @@ from pytest_mock import MockerFixture from frequenz.sdk import microgrid, timeseries -from frequenz.sdk.actor import ResamplerConfig from frequenz.sdk.microgrid import _power_distributing from frequenz.sdk.microgrid._power_distributing import ComponentPoolStatus from frequenz.sdk.microgrid._power_distributing._component_pool_status_tracker import ( ComponentPoolStatusTracker, ) +from frequenz.sdk.timeseries import ResamplerConfig from frequenz.sdk.timeseries.battery_pool.messages import BatteryPoolReport from ...utils.component_data_streamer import MockComponentDataStreamer diff --git a/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py b/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py index 89f29c808..10f25e7f3 100644 --- a/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py +++ b/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py @@ -17,9 +17,9 @@ from pytest_mock import MockerFixture from frequenz.sdk import microgrid -from frequenz.sdk.actor import ResamplerConfig from frequenz.sdk.microgrid import _power_distributing from frequenz.sdk.microgrid._data_pipeline import _DataPipeline +from frequenz.sdk.timeseries import ResamplerConfig from frequenz.sdk.timeseries.pv_pool import PVPoolReport from ...microgrid.fixtures import _Mocks diff --git a/tests/timeseries/mock_microgrid.py b/tests/timeseries/mock_microgrid.py index 3f6674b1b..326a4efa5 100644 --- a/tests/timeseries/mock_microgrid.py +++ b/tests/timeseries/mock_microgrid.py @@ -27,9 +27,9 @@ from frequenz.sdk import microgrid from frequenz.sdk._internal._asyncio import cancel_and_await -from frequenz.sdk.actor import ResamplerConfig from frequenz.sdk.microgrid import _data_pipeline from frequenz.sdk.microgrid.component_graph import _MicrogridComponentGraph +from frequenz.sdk.timeseries import ResamplerConfig from ..utils import MockMicrogridClient from ..utils.component_data_wrapper import ( From fc802d878fa0c2ccafaab76562f24cdbe05101f2 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 15:15:33 +0200 Subject: [PATCH 22/32] Use internal import to avoid circular import issues Signed-off-by: Leandro Lucarella --- src/frequenz/sdk/microgrid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frequenz/sdk/microgrid/__init__.py b/src/frequenz/sdk/microgrid/__init__.py index 9fefde947..5be3adc61 100644 --- a/src/frequenz/sdk/microgrid/__init__.py +++ b/src/frequenz/sdk/microgrid/__init__.py @@ -315,7 +315,7 @@ from datetime import timedelta -from ..actor import ResamplerConfig +from ..timeseries._resampling._config import ResamplerConfig from . import _data_pipeline, connection_manager from ._data_pipeline import ( consumer, From 99485744d451e722b4147f887cb93d9c2f7a124e Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 09:23:54 +0200 Subject: [PATCH 23/32] Add a new `ResamplerConfig2` that uses `WallClockTimerConfig` This configuration object is exactly the same as `ResamplerConfig` but replaces the `align_to` field with a new `timer_config` field that holds a `WallClockTimerConfig` instance (or `None` to use the default). It still inherits from `ResamplerConfig` to make the migration easier. Because the `ResamplingFunction` also takes the config, a `ResamplingFunction2` is also added to keep backwards compatibility with any custom resampling function users might be using. Signed-off-by: Leandro Lucarella --- src/frequenz/sdk/timeseries/__init__.py | 4 + .../sdk/timeseries/_resampling/_config.py | 173 +++++++++++++++++- 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/src/frequenz/sdk/timeseries/__init__.py b/src/frequenz/sdk/timeseries/__init__.py index cccfaaaed..d5bde1602 100644 --- a/src/frequenz/sdk/timeseries/__init__.py +++ b/src/frequenz/sdk/timeseries/__init__.py @@ -46,7 +46,9 @@ DEFAULT_BUFFER_LEN_MAX, DEFAULT_BUFFER_LEN_WARN, ResamplerConfig, + ResamplerConfig2, ResamplingFunction, + ResamplingFunction2, ) from ._resampling._exceptions import ResamplingError, SourceStoppedError from ._resampling._wall_clock_timer import ( @@ -67,8 +69,10 @@ "PeriodicFeatureExtractor", "ReceiverFetcher", "ResamplerConfig", + "ResamplerConfig2", "ResamplingError", "ResamplingFunction", + "ResamplingFunction2", "Sample", "Sample3Phase", "Sink", diff --git a/src/frequenz/sdk/timeseries/_resampling/_config.py b/src/frequenz/sdk/timeseries/_resampling/_config.py index 9be665c6e..047892803 100644 --- a/src/frequenz/sdk/timeseries/_resampling/_config.py +++ b/src/frequenz/sdk/timeseries/_resampling/_config.py @@ -8,13 +8,14 @@ import logging import statistics from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta from typing import Protocol from frequenz.core.datetime import UNIX_EPOCH from ._base_types import SourceProperties +from ._wall_clock_timer import WallClockTimerConfig _logger = logging.getLogger(__name__) @@ -210,3 +211,173 @@ def __post_init__(self) -> None: raise ValueError( f"align_to ({self.align_to}) should be a timezone aware datetime" ) + + +class ResamplingFunction2(Protocol): + """Combine multiple samples into a new one. + + A resampling function produces a new sample based on a list of pre-existing + samples. It can do "upsampling" when the data rate of the `input_samples` + period is smaller than the `resampling_period`, or "downsampling" if it is + bigger. + + In general, a resampling window is the same as the `resampling_period`, and + this function might receive input samples from multiple windows in the past to + enable extrapolation, but no samples from the future (so the timestamp of the + new sample that is going to be produced will always be bigger than the biggest + timestamp in the input data). + """ + + def __call__( + self, + input_samples: Sequence[tuple[datetime, float]], + resampler_config: ResamplerConfig | ResamplerConfig2, + source_properties: SourceProperties, + /, + ) -> float: + """Call the resampling function. + + Args: + input_samples: The sequence of pre-existing samples, where the first item is + the timestamp of the sample, and the second is the value of the sample. + The sequence must be non-empty. + resampler_config: The configuration of the resampler calling this + function. + source_properties: The properties of the source being resampled. + + Returns: + The value of new sample produced after the resampling. + """ + ... # pylint: disable=unnecessary-ellipsis + + +@dataclass(frozen=True) +class ResamplerConfig2(ResamplerConfig): + """Resampler configuration.""" + + resampling_period: timedelta + """The resampling period. + + This is the time it passes between resampled data should be calculated. + + It must be a positive time span. + """ + + max_data_age_in_periods: float = 3.0 + """The maximum age a sample can have to be considered *relevant* for resampling. + + Expressed in number of periods, where period is the `resampling_period` + if we are downsampling (resampling period bigger than the input period) or + the *input sampling period* if we are upsampling (input period bigger than + the resampling period). + + It must be bigger than 1.0. + + Example: + If `resampling_period` is 3 seconds, the input sampling period is + 1 and `max_data_age_in_periods` is 2, then data older than 3*2 + = 6 seconds will be discarded when creating a new sample and never + passed to the resampling function. + + If `resampling_period` is 3 seconds, the input sampling period is + 5 and `max_data_age_in_periods` is 2, then data older than 5*2 + = 10 seconds will be discarded when creating a new sample and never + passed to the resampling function. + """ + + resampling_function: ResamplingFunction2 = lambda samples, _, __: statistics.fmean( + s[1] for s in samples + ) + """The resampling function. + + This function will be applied to the sequence of relevant samples at + a given time. The result of the function is what is sent as the resampled + value. + """ + + initial_buffer_len: int = DEFAULT_BUFFER_LEN_INIT + """The initial length of the resampling buffer. + + The buffer could grow or shrink depending on the source properties, + like sampling rate, to make sure all the requested past sampling periods + can be stored. + + It must be at least 1 and at most `max_buffer_len`. + """ + + warn_buffer_len: int = DEFAULT_BUFFER_LEN_WARN + """The minimum length of the resampling buffer that will emit a warning. + + If a buffer grows bigger than this value, it will emit a warning in the + logs, so buffers don't grow too big inadvertently. + + It must be at least 1 and at most `max_buffer_len`. + """ + + max_buffer_len: int = DEFAULT_BUFFER_LEN_MAX + """The maximum length of the resampling buffer. + + Buffers won't be allowed to grow beyond this point even if it would be + needed to keep all the requested past sampling periods. An error will be + emitted in the logs if the buffer length needs to be truncated to this + value. + + It must be at bigger than `warn_buffer_len`. + """ + + align_to: datetime | None = field(default=None, init=False) + """Deprecated: Use timer_config.align_to instead.""" + + timer_config: WallClockTimerConfig | None = None + """The custom configuration of the wall clock timer used to keep track of time. + + If not provided or `None`, a configuration will be created by passing the + [`resampling_period`][frequenz.sdk.timeseries.ResamplerConfig2.resampling_period] to + the [`from_interval()`][frequenz.sdk.timeseries.WallClockTimerConfig.from_interval] + method. + """ + + def __post_init__(self) -> None: + """Check that config values are valid. + + Raises: + ValueError: If any value is out of range. + """ + if self.resampling_period.total_seconds() < 0.0: + raise ValueError( + f"resampling_period ({self.resampling_period}) must be positive" + ) + if self.max_data_age_in_periods < 1.0: + raise ValueError( + f"max_data_age_in_periods ({self.max_data_age_in_periods}) should be at least 1.0" + ) + if self.warn_buffer_len < 1: + raise ValueError( + f"warn_buffer_len ({self.warn_buffer_len}) should be at least 1" + ) + if self.max_buffer_len <= self.warn_buffer_len: + raise ValueError( + f"max_buffer_len ({self.max_buffer_len}) should " + f"be bigger than warn_buffer_len ({self.warn_buffer_len})" + ) + + if self.initial_buffer_len < 1: + raise ValueError( + f"initial_buffer_len ({self.initial_buffer_len}) should at least 1" + ) + if self.initial_buffer_len > self.max_buffer_len: + raise ValueError( + f"initial_buffer_len ({self.initial_buffer_len}) is bigger " + f"than max_buffer_len ({self.max_buffer_len}), use a smaller " + "initial_buffer_len or a bigger max_buffer_len" + ) + if self.initial_buffer_len > self.warn_buffer_len: + _logger.warning( + "initial_buffer_len (%s) is bigger than warn_buffer_len (%s)", + self.initial_buffer_len, + self.warn_buffer_len, + ) + if self.align_to is not None: + raise ValueError( + f"align_to ({self.align_to}) must be specified via timer_config" + ) From 872e86f20a3a765be4af912a23f3c947faa248dd Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 10:31:53 +0200 Subject: [PATCH 24/32] Add support for `WallClockTimer` to the `Resampler` When passing a `ResamplerConfig2` to the resampler, it will use a `WallClockTimer` instead of a monotonic `Timer` to trigger the resampling. Signed-off-by: Leandro Lucarella --- .../sdk/timeseries/_resampling/_resampler.py | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/frequenz/sdk/timeseries/_resampling/_resampler.py b/src/frequenz/sdk/timeseries/_resampling/_resampler.py index f779f8ed3..81471fc0d 100644 --- a/src/frequenz/sdk/timeseries/_resampling/_resampler.py +++ b/src/frequenz/sdk/timeseries/_resampling/_resampler.py @@ -12,6 +12,7 @@ from bisect import bisect from collections import deque from datetime import datetime, timedelta, timezone +from typing import assert_never from frequenz.channels.timer import Timer, TriggerAllMissed, _to_microseconds from frequenz.quantities import Quantity @@ -19,8 +20,9 @@ from ..._internal._asyncio import cancel_and_await from .._base_types import Sample from ._base_types import Sink, Source, SourceProperties -from ._config import ResamplerConfig +from ._config import ResamplerConfig, ResamplerConfig2 from ._exceptions import ResamplingError, SourceStoppedError +from ._wall_clock_timer import TickInfo, WallClockTimer _logger = logging.getLogger(__name__) @@ -44,7 +46,10 @@ def __init__(self, config: ResamplerConfig) -> None: """Initialize an instance. Args: - config: The configuration for the resampler. + config: The configuration for the resampler. If a `ResamplerConfig2` is + provided, the resampler will use a + [`WallClockTimer`][frequenz.sdk.timeseries.WallClockTimer] instead of a + [`Timer`][frequenz.channels.timer.Timer]. """ self._config = config """The configuration for this resampler.""" @@ -52,6 +57,13 @@ def __init__(self, config: ResamplerConfig) -> None: self._resamplers: dict[Source, _StreamingHelper] = {} """A mapping between sources and the streaming helper handling that source.""" + self._timer: Timer | WallClockTimer + """The timer used to trigger the resampling windows.""" + + if isinstance(config, ResamplerConfig2): + self._timer = WallClockTimer(config.resampling_period, config.timer_config) + return + window_end, start_delay_time = self._calculate_window_end() self._window_end: datetime = window_end """The time in which the current window ends. @@ -66,7 +78,7 @@ def __init__(self, config: ResamplerConfig) -> None: the window end is deterministic. """ - self._timer: Timer = Timer(config.resampling_period, TriggerAllMissed()) + self._timer = Timer(config.resampling_period, TriggerAllMissed()) """The timer used to trigger the resampling windows.""" # Hack to align the timer, this should be implemented in the Timer class @@ -162,20 +174,31 @@ async def resample(self, *, one_shot: bool = False) -> None: seconds=self._config.resampling_period.total_seconds() / 10.0 ) - async for drift in self._timer: - now = datetime.now(tz=timezone.utc) - - if drift > tolerance: - _logger.warning( - "The resampling task woke up too late. Resampling should have " - "started at %s, but it started at %s (tolerance: %s, " - "difference: %s; resampling period: %s)", - self._window_end, - now, - tolerance, - drift, - self._config.resampling_period, - ) + async for tick_info in self._timer: + next_tick_time: datetime + match tick_info: + case TickInfo(): # WallClockTimer + next_tick_time = tick_info.expected_tick_time + + case timedelta() as drift: # Timer (monotonic) + next_tick_time = self._window_end + + if drift > tolerance: + _logger.warning( + "The resampling task woke up too late. Resampling should have " + "started at %s, but it started at %s (tolerance: %s, " + "difference: %s; resampling period: %s)", + self._window_end, + datetime.now(tz=timezone.utc), + tolerance, + tick_info, + self._config.resampling_period, + ) + + self._window_end += self._config.resampling_period + + case unexpected: + assert_never(unexpected) # We need to make a copy here because we need to match the results to the # current resamplers, and since we await here, new resamplers could be added @@ -183,11 +206,10 @@ async def resample(self, *, one_shot: bool = False) -> None: # cause the results to be out of sync. resampler_sources = list(self._resamplers) results = await asyncio.gather( - *[r.resample(self._window_end) for r in self._resamplers.values()], + *[r.resample(next_tick_time) for r in self._resamplers.values()], return_exceptions=True, ) - self._window_end += self._config.resampling_period exceptions = { source: result for source, result in zip(resampler_sources, results) From 5380edce640c5d323d0b4fc63eef4999efa9f9f1 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 11:09:28 +0200 Subject: [PATCH 25/32] Import from public modules when possible Signed-off-by: Leandro Lucarella --- tests/timeseries/test_resampling.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/timeseries/test_resampling.py b/tests/timeseries/test_resampling.py index dac74af86..7fbaf5a66 100644 --- a/tests/timeseries/test_resampling.py +++ b/tests/timeseries/test_resampling.py @@ -16,17 +16,15 @@ from frequenz.channels import Broadcast, SenderError from frequenz.quantities import Quantity -from frequenz.sdk.timeseries import Sample -from frequenz.sdk.timeseries._resampling._base_types import ( - Sink, - Source, - SourceProperties, -) -from frequenz.sdk.timeseries._resampling._config import ( +from frequenz.sdk.timeseries import ( DEFAULT_BUFFER_LEN_MAX, DEFAULT_BUFFER_LEN_WARN, ResamplerConfig, ResamplingFunction, + Sample, + Sink, + Source, + SourceProperties, ) from frequenz.sdk.timeseries._resampling._exceptions import ( ResamplingError, From 3f5a70d483bfee8a6b20f141942c2ef55ecf9057 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 11:12:31 +0200 Subject: [PATCH 26/32] Make disabling of pylint checks local When using `# pylint: disable=...` in its own line, it is disabled for the rest of the scope, not only for the next line. Signed-off-by: Leandro Lucarella --- tests/timeseries/test_resampling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/timeseries/test_resampling.py b/tests/timeseries/test_resampling.py index 7fbaf5a66..10a19a01e 100644 --- a/tests/timeseries/test_resampling.py +++ b/tests/timeseries/test_resampling.py @@ -239,7 +239,7 @@ async def test_calculate_window_end_trivial_cases( ) ) fake_time.move_to(now) - # pylint: disable=protected-access + # pylint: disable-next=protected-access assert resampler._calculate_window_end() == result # Repeat the test with align_to=None, so the result should be align to now @@ -1406,7 +1406,7 @@ async def test_resampling_all_zeros( def _get_buffer_len(resampler: Resampler, source_receiver: Source) -> int: - # pylint: disable=protected-access + # pylint: disable-next=protected-access blen = resampler._resamplers[source_receiver]._helper._buffer.maxlen assert blen is not None return blen From 1b7b3c7625e8f1eb7edf81aa0a800af33ddcdbe4 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 12:46:00 +0200 Subject: [PATCH 27/32] Add tests for the resampler using `ResampleConfig2` We just parametrize the existing tests to also test the resampler when using the `ResamplerConfig2` class (i.e. the wall clock timer). Signed-off-by: Leandro Lucarella --- tests/timeseries/test_resampling.py | 181 +++++++++++++++++++++------- 1 file changed, 140 insertions(+), 41 deletions(-) diff --git a/tests/timeseries/test_resampling.py b/tests/timeseries/test_resampling.py index 10a19a01e..3629b5f4e 100644 --- a/tests/timeseries/test_resampling.py +++ b/tests/timeseries/test_resampling.py @@ -20,6 +20,7 @@ DEFAULT_BUFFER_LEN_MAX, DEFAULT_BUFFER_LEN_WARN, ResamplerConfig, + ResamplerConfig2, ResamplingFunction, Sample, Sink, @@ -98,12 +99,14 @@ async def _assert_no_more_samples( @pytest.mark.parametrize("init_len", list(range(1, DEFAULT_BUFFER_LEN_WARN + 1, 16))) +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampler_config_len_ok( init_len: int, + config_class: type[ResamplerConfig], caplog: pytest.LogCaptureFixture, ) -> None: """Test checks on the resampling buffer.""" - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=1.0), initial_buffer_len=init_len, ) @@ -116,11 +119,12 @@ async def test_resampler_config_len_ok( "init_len", range(DEFAULT_BUFFER_LEN_WARN + 1, DEFAULT_BUFFER_LEN_MAX + 1, 64), ) +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampler_config_len_warn( - init_len: int, caplog: pytest.LogCaptureFixture + init_len: int, config_class: type[ResamplerConfig], caplog: pytest.LogCaptureFixture ) -> None: """Test checks on the resampling buffer.""" - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=1.0), initial_buffer_len=init_len, ) @@ -142,21 +146,26 @@ async def test_resampler_config_len_warn( "init_len", list(range(-2, 1)) + [DEFAULT_BUFFER_LEN_MAX + 1, DEFAULT_BUFFER_LEN_MAX + 2], ) -async def test_resampler_config_len_error(init_len: int) -> None: +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) +async def test_resampler_config_len_error( + init_len: int, config_class: type[ResamplerConfig] +) -> None: """Test checks on the resampling buffer.""" with pytest.raises(ValueError): - _ = ResamplerConfig( + _ = config_class( resampling_period=timedelta(seconds=1.0), initial_buffer_len=init_len, ) +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_helper_buffer_too_big( + config_class: type[ResamplerConfig], fake_time: time_machine.Coordinates, caplog: pytest.LogCaptureFixture, ) -> None: """Test checks on the resampling buffer.""" - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=DEFAULT_BUFFER_LEN_MAX + 1), max_data_age_in_periods=1, ) @@ -263,8 +272,11 @@ async def test_calculate_window_end_trivial_cases( assert none_result[0] == now + resampling_period +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampling_window_size_is_constant( - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: """Test resampling window size is consistent.""" timestamp = datetime.now(timezone.utc) @@ -275,7 +287,7 @@ async def test_resampling_window_size_is_constant( resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=1.0, resampling_function=resampling_fun_mock, @@ -357,6 +369,9 @@ async def test_resampling_window_size_is_constant( resampling_fun_mock.reset_mock() +# Not parametrized because now warnings are handled by the wall clock timer when the +# wall clock timer is used, not the resampler, so it should be tested in the wall clock +# timer tests. async def test_timer_errors_are_logged( # pylint: disable=too-many-statements fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]], @@ -517,10 +532,13 @@ async def test_timer_errors_are_logged( # pylint: disable=too-many-statements resampling_fun_mock.reset_mock() +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_future_samples_not_included( - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: - """Test resampling window size is consistent.""" + """Test that future samples are not included in the resampling.""" timestamp = datetime.now(timezone.utc) resampling_period_s = 2 @@ -529,7 +547,7 @@ async def test_future_samples_not_included( resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=2.0, resampling_function=resampling_fun_mock, @@ -612,8 +630,11 @@ async def test_future_samples_not_included( ) +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampling_with_one_window( - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: """Test resampling with one resampling window (saving samples of the last period only).""" timestamp = datetime.now(timezone.utc) @@ -624,7 +645,7 @@ async def test_resampling_with_one_window( resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=1.0, resampling_function=resampling_fun_mock, @@ -736,8 +757,11 @@ async def test_resampling_with_one_window( # Even when a lot could be refactored to use smaller functions, I'm allowing # too many statements because it makes following failures in tests more easy # when the code is very flat. +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampling_with_one_and_a_half_windows( # pylint: disable=too-many-statements - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: """Test resampling with 1.5 resampling windows.""" timestamp = datetime.now(timezone.utc) @@ -748,7 +772,7 @@ async def test_resampling_with_one_and_a_half_windows( # pylint: disable=too-ma resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=1.5, resampling_function=resampling_fun_mock, @@ -915,8 +939,11 @@ async def test_resampling_with_one_and_a_half_windows( # pylint: disable=too-ma # Even when a lot could be refactored to use smaller functions, I'm allowing # too many statements because it makes following failures in tests more easy # when the code is very flat. +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampling_with_two_windows( # pylint: disable=too-many-statements - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: """Test resampling with 2 resampling windows.""" timestamp = datetime.now(timezone.utc) @@ -927,7 +954,7 @@ async def test_resampling_with_two_windows( # pylint: disable=too-many-statemen resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=2.0, resampling_function=resampling_fun_mock, @@ -1092,8 +1119,11 @@ async def test_resampling_with_two_windows( # pylint: disable=too-many-statemen assert _get_buffer_len(resampler, source_receiver) == config.initial_buffer_len +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_receiving_stopped_resampling_error( - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: """Test resampling errors if a receiver stops.""" timestamp = datetime.now(timezone.utc) @@ -1104,7 +1134,7 @@ async def test_receiving_stopped_resampling_error( resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=2.0, resampling_function=resampling_fun_mock, @@ -1155,7 +1185,10 @@ async def test_receiving_stopped_resampling_error( assert timeseries_error.source is source_receiver -async def test_receiving_resampling_error(fake_time: time_machine.Coordinates) -> None: +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) +async def test_receiving_resampling_error( + config_class: type[ResamplerConfig], fake_time: time_machine.Coordinates +) -> None: """Test resampling stops if there is an unknown error.""" timestamp = datetime.now(timezone.utc) @@ -1166,7 +1199,7 @@ async def test_receiving_resampling_error(fake_time: time_machine.Coordinates) - spec=ResamplingFunction, return_value=expected_resampled_value ) resampler = Resampler( - ResamplerConfig( + config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=2.0, resampling_function=resampling_fun_mock, @@ -1200,21 +1233,28 @@ async def make_fake_source() -> Source: assert isinstance(timeseries_error, TestException) +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_timer_is_aligned( + config_class: type[ResamplerConfig], fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]], caplog: pytest.LogCaptureFixture, ) -> None: - """Test that big differences between the expected window end and the fired timer are logged.""" - timestamp = datetime.now(timezone.utc) - + """Test that the resampling timer is aligned to the resampling period.""" resampling_period_s = 2 expected_resampled_value = 42.0 + # There is a small difference in behaviour between the (monotonic) Timer and the + # WallClockTimer: the former will wait until it has at least one full period filled + # with data, while the later fires at the next aligned period after it started. + is_wall_clock = issubclass(config_class, ResamplerConfig2) + start_offset_s = 0 if is_wall_clock else resampling_period_s + timestamp = datetime.now(timezone.utc) + resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=2.0, resampling_function=resampling_fun_mock, @@ -1224,6 +1264,7 @@ async def test_timer_is_aligned( # Advance the time a bit so that the resampler is not aligned to the resampling # period await _advance_time(fake_time, resampling_period_s / 3) + # t = 0.667s resampler = Resampler(config) @@ -1236,12 +1277,15 @@ async def test_timer_is_aligned( source_props = resampler.get_source_properties(source_receiver) # Test timeline - # start delay timer start + # Timer start delay timer start first resampling # ,-------------|---------------------| - # start = 0.667 + # start = 0.667 | | # t(s) 0 | 1 1.5 2 2.5 3 4 # |-------+--|-----|----|----|-----|----------R-----> (no more samples) - # value 5.0 12.0 2.0 4.0 5.0 + # input sample | 5.0 12.0 2.0 4.0 5.0 + # `-------------|---------------------| + # WallClockTimer timer start first resampling second resampling + # (no extra start delay) # # R = resampling is done @@ -1256,28 +1300,80 @@ async def test_timer_is_aligned( await source_sender.send(sample2_5s) await source_sender.send(sample3s) await source_sender.send(sample4s) - await _advance_time(fake_time, resampling_period_s * (1 + 2 / 3)) + await _advance_time(fake_time, start_offset_s + resampling_period_s * 2 / 3) + # t = 2 (wall clock) / 4 (mono) await resampler.resample(one_shot=True) - assert datetime.now(timezone.utc).timestamp() == pytest.approx(4) - assert asyncio.get_running_loop().time() == pytest.approx(4) + assert datetime.now(timezone.utc).timestamp() == pytest.approx( + 2 if is_wall_clock else 4 + ) + assert asyncio.get_running_loop().time() == pytest.approx(2 if is_wall_clock else 4) sink_mock.assert_called_once_with( Sample( - timestamp + timedelta(seconds=resampling_period_s * 2), + timestamp + timedelta(seconds=start_offset_s + resampling_period_s), Quantity(expected_resampled_value), ) ) - resampling_fun_mock.assert_called_once_with( - a_sequence( - as_float_tuple(sample1s), - as_float_tuple(sample1_5s), + # We are using a buffer of 2 windows, so when the monotonic timer is used, which + # fires for the first time one period later, we will use all samples received so far + # (including the ones for the first period where it didn't fired), for the wall clock + # timer, we will use only the samples received in the first period. + expected_samples: list[tuple[datetime, float]] = [ + as_float_tuple(sample1s), + as_float_tuple(sample1_5s), + ] + if not is_wall_clock: + expected_samples.extend( + [ + as_float_tuple(sample2_5s), + as_float_tuple(sample3s), + as_float_tuple(sample4s), + ] + ) + resampling_fun_mock.assert_called_once_with(expected_samples, config, source_props) + assert not [ + *_filter_logs( + caplog.record_tuples, + logger_level=logging.WARNING, + ) + ] + sink_mock.reset_mock() + resampling_fun_mock.reset_mock() + + await _advance_time(fake_time, resampling_period_s) + # t = 4 (wall clock) / 6 (mono) + await resampler.resample(one_shot=True) + + assert datetime.now(timezone.utc).timestamp() == pytest.approx( + 4 if is_wall_clock else 6 + ) + assert asyncio.get_running_loop().time() == pytest.approx(4 if is_wall_clock else 6) + sink_mock.assert_called_once_with( + Sample( + timestamp + timedelta(seconds=start_offset_s + resampling_period_s * 2), + Quantity(expected_resampled_value), + ) + ) + # We are using a buffer of 2 windows, so when the monotonic timer is used so it is + # fired at 6 seconds, we only have in the buffer the samples for the window 2-4 + # seconds. For the wall clock timer, which fires at 4 seconds, we have + # the samples for the last 2 periods, the time window 0-4 seconds. + expected_samples = [] + if is_wall_clock: + expected_samples.extend( + [ + as_float_tuple(sample1s), + as_float_tuple(sample1_5s), + ] + ) + expected_samples.extend( + [ as_float_tuple(sample2_5s), as_float_tuple(sample3s), as_float_tuple(sample4s), - ), - config, - source_props, + ] ) + resampling_fun_mock.assert_called_once_with(expected_samples, config, source_props) assert not [ *_filter_logs( caplog.record_tuples, @@ -1288,8 +1384,11 @@ async def test_timer_is_aligned( resampling_fun_mock.reset_mock() +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_resampling_all_zeros( - fake_time: time_machine.Coordinates, source_chan: Broadcast[Sample[Quantity]] + config_class: type[ResamplerConfig], + fake_time: time_machine.Coordinates, + source_chan: Broadcast[Sample[Quantity]], ) -> None: """Test resampling with one resampling window full of zeros.""" timestamp = datetime.now(timezone.utc) @@ -1300,7 +1399,7 @@ async def test_resampling_all_zeros( resampling_fun_mock = MagicMock( spec=ResamplingFunction, return_value=expected_resampled_value ) - config = ResamplerConfig( + config = config_class( resampling_period=timedelta(seconds=resampling_period_s), max_data_age_in_periods=1.0, resampling_function=resampling_fun_mock, From 0a4d1010c5c6c33593eadc24cc76f2b0af93d1b9 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 15:05:52 +0200 Subject: [PATCH 28/32] Use the new wall clock timer in benchmarks Signed-off-by: Leandro Lucarella --- benchmarks/power_distribution/power_distributor.py | 4 ++-- benchmarks/timeseries/resampling.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/power_distribution/power_distributor.py b/benchmarks/power_distribution/power_distributor.py index fd15466a6..a1eb5edd3 100644 --- a/benchmarks/power_distribution/power_distributor.py +++ b/benchmarks/power_distribution/power_distributor.py @@ -17,7 +17,6 @@ from frequenz.quantities import Power from frequenz.sdk import microgrid -from frequenz.sdk.actor import ResamplerConfig from frequenz.sdk.microgrid import connection_manager from frequenz.sdk.microgrid._power_distributing import ( ComponentPoolStatus, @@ -29,6 +28,7 @@ Result, Success, ) +from frequenz.sdk.timeseries import ResamplerConfig2 HOST = "microgrid.sandbox.api.frequenz.io" PORT = 62060 @@ -140,7 +140,7 @@ async def run() -> None: """Create microgrid api and run tests.""" await microgrid.initialize( "grpc://microgrid.sandbox.api.frequenz.io:62060", - ResamplerConfig(resampling_period=timedelta(seconds=1.0)), + ResamplerConfig2(resampling_period=timedelta(seconds=1.0)), ) all_batteries: set[Component] = connection_manager.get().component_graph.components( diff --git a/benchmarks/timeseries/resampling.py b/benchmarks/timeseries/resampling.py index 58d360c9c..673e73aef 100644 --- a/benchmarks/timeseries/resampling.py +++ b/benchmarks/timeseries/resampling.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta, timezone from timeit import timeit -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig, ResamplerConfig2 from frequenz.sdk.timeseries._resampling._base_types import SourceProperties from frequenz.sdk.timeseries._resampling._resampler import _ResamplingHelper @@ -25,7 +25,7 @@ def _benchmark_resampling_helper(resamples: int, samples: int) -> None: """Benchmark the resampling helper.""" helper = _ResamplingHelper( "benchmark", - ResamplerConfig( + ResamplerConfig2( resampling_period=timedelta(seconds=1.0), max_data_age_in_periods=3.0, resampling_function=nop, From 18fbebfe64c196f9948effa83affba87c4d88d2a Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 15:16:17 +0200 Subject: [PATCH 29/32] Update documentation to use the new `ResamperConfig2` We want to encourage new users to use the new wall clock timer, so it better to use the new config object in examples. Signed-off-by: Leandro Lucarella --- docs/tutorials/getting_started.md | 8 ++++---- examples/battery_pool.py | 4 ++-- src/frequenz/sdk/timeseries/_voltage_streamer.py | 4 ++-- src/frequenz/sdk/timeseries/consumer.py | 4 ++-- src/frequenz/sdk/timeseries/grid.py | 4 ++-- .../sdk/timeseries/logical_meter/_logical_meter.py | 4 ++-- src/frequenz/sdk/timeseries/producer.py | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index d8620c8c1..ddeed9d00 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -32,7 +32,7 @@ import asyncio from datetime import timedelta from frequenz.sdk import microgrid -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 ``` ## Create the application skeleton @@ -49,7 +49,7 @@ async def run() -> None: # Initialize the microgrid await microgrid.initialize( server_url, - ResamplerConfig(resampling_period=timedelta(seconds=1)), + ResamplerConfig2(resampling_period=timedelta(seconds=1)), ) # Define your application logic here @@ -100,7 +100,7 @@ import asyncio from datetime import timedelta from frequenz.sdk import microgrid -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 async def run() -> None: # This points to the default Frequenz microgrid sandbox @@ -109,7 +109,7 @@ async def run() -> None: # Initialize the microgrid await microgrid.initialize( server_url, - ResamplerConfig(resampling_period=timedelta(seconds=1)), + ResamplerConfig2(resampling_period=timedelta(seconds=1)), ) # Define your application logic here diff --git a/examples/battery_pool.py b/examples/battery_pool.py index 91070cb4c..cae09c622 100644 --- a/examples/battery_pool.py +++ b/examples/battery_pool.py @@ -11,7 +11,7 @@ from frequenz.channels import merge from frequenz.sdk import microgrid -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 MICROGRID_API_URL = "grpc://microgrid.sandbox.api.frequenz.io:62060" @@ -24,7 +24,7 @@ async def main() -> None: await microgrid.initialize( MICROGRID_API_URL, - resampler_config=ResamplerConfig(resampling_period=timedelta(seconds=1.0)), + resampler_config=ResamplerConfig2(resampling_period=timedelta(seconds=1.0)), ) battery_pool = microgrid.new_battery_pool(priority=5) diff --git a/src/frequenz/sdk/timeseries/_voltage_streamer.py b/src/frequenz/sdk/timeseries/_voltage_streamer.py index 457ea03f9..6329117ab 100644 --- a/src/frequenz/sdk/timeseries/_voltage_streamer.py +++ b/src/frequenz/sdk/timeseries/_voltage_streamer.py @@ -35,11 +35,11 @@ class VoltageStreamer: from datetime import timedelta from frequenz.sdk import microgrid - from frequenz.sdk.timeseries import ResamplerConfig + from frequenz.sdk.timeseries import ResamplerConfig2 await microgrid.initialize( "grpc://127.0.0.1:50051", - ResamplerConfig(resampling_period=timedelta(seconds=1)) + ResamplerConfig2(resampling_period=timedelta(seconds=1)) ) # Get a receiver for the phase-to-neutral voltage. diff --git a/src/frequenz/sdk/timeseries/consumer.py b/src/frequenz/sdk/timeseries/consumer.py index 9b85b3c2a..71f29e511 100644 --- a/src/frequenz/sdk/timeseries/consumer.py +++ b/src/frequenz/sdk/timeseries/consumer.py @@ -36,11 +36,11 @@ class Consumer: from datetime import timedelta from frequenz.sdk import microgrid - from frequenz.sdk.timeseries import ResamplerConfig + from frequenz.sdk.timeseries import ResamplerConfig2 await microgrid.initialize( "grpc://127.0.0.1:50051", - ResamplerConfig(resampling_period=timedelta(seconds=1.0)) + ResamplerConfig2(resampling_period=timedelta(seconds=1.0)) ) consumer = microgrid.consumer() diff --git a/src/frequenz/sdk/timeseries/grid.py b/src/frequenz/sdk/timeseries/grid.py index 72ff65f1c..e6bc97b43 100644 --- a/src/frequenz/sdk/timeseries/grid.py +++ b/src/frequenz/sdk/timeseries/grid.py @@ -46,11 +46,11 @@ class Grid: from datetime import timedelta from frequenz.sdk import microgrid - from frequenz.sdk.timeseries import ResamplerConfig + from frequenz.sdk.timeseries import ResamplerConfig2 await microgrid.initialize( "grpc://127.0.0.1:50051", - ResamplerConfig(resampling_period=timedelta(seconds=1)) + ResamplerConfig2(resampling_period=timedelta(seconds=1)) ) grid = microgrid.grid() diff --git a/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py b/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py index b7b20f059..833ed9cb6 100644 --- a/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py +++ b/src/frequenz/sdk/timeseries/logical_meter/_logical_meter.py @@ -33,13 +33,13 @@ class LogicalMeter: from datetime import timedelta from frequenz.sdk import microgrid - from frequenz.sdk.timeseries import ResamplerConfig + from frequenz.sdk.timeseries import ResamplerConfig2 from frequenz.client.microgrid import ComponentMetricId await microgrid.initialize( "grpc://microgrid.sandbox.api.frequenz.io:62060", - ResamplerConfig(resampling_period=timedelta(seconds=1)), + ResamplerConfig2(resampling_period=timedelta(seconds=1)), ) logical_meter = ( diff --git a/src/frequenz/sdk/timeseries/producer.py b/src/frequenz/sdk/timeseries/producer.py index 4e5103399..0e65f5711 100644 --- a/src/frequenz/sdk/timeseries/producer.py +++ b/src/frequenz/sdk/timeseries/producer.py @@ -36,11 +36,11 @@ class Producer: from datetime import timedelta from frequenz.sdk import microgrid - from frequenz.sdk.timeseries import ResamplerConfig + from frequenz.sdk.timeseries import ResamplerConfig2 await microgrid.initialize( "grpc://127.0.0.1:50051", - ResamplerConfig(resampling_period=timedelta(seconds=1.0)) + ResamplerConfig2(resampling_period=timedelta(seconds=1.0)) ) producer = microgrid.producer() From 08814d60ecac4cc0f81752d53a43f713d476bb09 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 15:45:21 +0200 Subject: [PATCH 30/32] Use `ResampleConfig2` in other tests In tests where there are no behavior differences between using the old monotonic timer and the new wall clock timer, we just use the wall clock timer, as we plan to deprecate the other timer soon. Signed-off-by: Leandro Lucarella --- tests/actor/test_resampling.py | 6 +++--- tests/microgrid/fixtures.py | 4 ++-- tests/microgrid/test_datapipeline.py | 4 ++-- tests/timeseries/_battery_pool/test_battery_pool.py | 6 +++--- .../_battery_pool/test_battery_pool_control_methods.py | 4 ++-- .../test_ev_charger_pool_control_methods.py | 4 ++-- tests/timeseries/_pv_pool/test_pv_pool_control_methods.py | 4 ++-- tests/timeseries/mock_microgrid.py | 4 ++-- tests/timeseries/mock_resampler.py | 4 ++-- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/actor/test_resampling.py b/tests/actor/test_resampling.py index 587f72568..17406dd79 100644 --- a/tests/actor/test_resampling.py +++ b/tests/actor/test_resampling.py @@ -17,7 +17,7 @@ from frequenz.sdk._internal._channels import ChannelRegistry from frequenz.sdk.microgrid._data_sourcing import ComponentMetricRequest from frequenz.sdk.microgrid._resampling import ComponentMetricsResamplingActor -from frequenz.sdk.timeseries import ResamplerConfig, Sample +from frequenz.sdk.timeseries import ResamplerConfig2, Sample @pytest.fixture(autouse=True) @@ -113,7 +113,7 @@ async def test_single_request( channel_registry=channel_registry, data_sourcing_request_sender=data_source_req_chan.new_sender(), resampling_request_receiver=resampling_req_chan.new_receiver(), - config=ResamplerConfig( + config=ResamplerConfig2( resampling_period=timedelta(seconds=0.2), max_data_age_in_periods=2, ), @@ -156,7 +156,7 @@ async def test_duplicate_request( channel_registry=channel_registry, data_sourcing_request_sender=data_source_req_chan.new_sender(), resampling_request_receiver=resampling_req_chan.new_receiver(), - config=ResamplerConfig( + config=ResamplerConfig2( resampling_period=timedelta(seconds=0.2), max_data_age_in_periods=2, ), diff --git a/tests/microgrid/fixtures.py b/tests/microgrid/fixtures.py index 3d0ac0e6b..8ce54a5dc 100644 --- a/tests/microgrid/fixtures.py +++ b/tests/microgrid/fixtures.py @@ -18,7 +18,7 @@ from frequenz.sdk import microgrid from frequenz.sdk.microgrid._power_distributing import ComponentPoolStatus from frequenz.sdk.microgrid.component_graph import _MicrogridComponentGraph -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 from ..timeseries.mock_microgrid import MockMicrogrid from ..utils.component_data_streamer import MockComponentDataStreamer @@ -55,7 +55,7 @@ async def new( if microgrid._data_pipeline._DATA_PIPELINE is not None: microgrid._data_pipeline._DATA_PIPELINE = None await microgrid._data_pipeline.initialize( - ResamplerConfig(resampling_period=timedelta(seconds=0.1)) + ResamplerConfig2(resampling_period=timedelta(seconds=0.1)) ) streamer = MockComponentDataStreamer(mockgrid.mock_client) diff --git a/tests/microgrid/test_datapipeline.py b/tests/microgrid/test_datapipeline.py index 7a30163a6..62bc8b8a2 100644 --- a/tests/microgrid/test_datapipeline.py +++ b/tests/microgrid/test_datapipeline.py @@ -19,7 +19,7 @@ from pytest_mock import MockerFixture from frequenz.sdk.microgrid._data_pipeline import _DataPipeline -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 from ..utils.mock_microgrid_client import MockMicrogridClient @@ -36,7 +36,7 @@ async def test_actors_started( ) -> None: """Test that the datasourcing, resampling and power distributing actors are started.""" datapipeline = _DataPipeline( - resampler_config=ResamplerConfig(resampling_period=timedelta(seconds=1)) + resampler_config=ResamplerConfig2(resampling_period=timedelta(seconds=1)) ) await asyncio.sleep(1) diff --git a/tests/timeseries/_battery_pool/test_battery_pool.py b/tests/timeseries/_battery_pool/test_battery_pool.py index fa4b6910d..8bbda43c6 100644 --- a/tests/timeseries/_battery_pool/test_battery_pool.py +++ b/tests/timeseries/_battery_pool/test_battery_pool.py @@ -34,7 +34,7 @@ from frequenz.sdk.microgrid._power_distributing._component_managers._battery_manager import ( _get_battery_inverter_mappings, ) -from frequenz.sdk.timeseries import Bounds, ResamplerConfig, Sample +from frequenz.sdk.timeseries import Bounds, ResamplerConfig2, Sample from frequenz.sdk.timeseries._base_types import SystemBounds from frequenz.sdk.timeseries.battery_pool import BatteryPool from frequenz.sdk.timeseries.formula_engine._formula_generators._formula_generator import ( @@ -138,7 +138,7 @@ async def setup_all_batteries(mocker: MockerFixture) -> AsyncIterator[SetupArgs] # pylint: disable=protected-access microgrid._data_pipeline._DATA_PIPELINE = None await microgrid._data_pipeline.initialize( - ResamplerConfig(resampling_period=timedelta(seconds=min_update_interval)) + ResamplerConfig2(resampling_period=timedelta(seconds=min_update_interval)) ) streamer = MockComponentDataStreamer(mock_microgrid) @@ -191,7 +191,7 @@ async def setup_batteries_pool(mocker: MockerFixture) -> AsyncIterator[SetupArgs # pylint: disable=protected-access microgrid._data_pipeline._DATA_PIPELINE = None await microgrid._data_pipeline.initialize( - ResamplerConfig(resampling_period=timedelta(seconds=min_update_interval)) + ResamplerConfig2(resampling_period=timedelta(seconds=min_update_interval)) ) # We don't use status channel from the sdk interface to limit diff --git a/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py b/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py index da54139ad..9837bc382 100644 --- a/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py +++ b/tests/timeseries/_battery_pool/test_battery_pool_control_methods.py @@ -21,7 +21,7 @@ from frequenz.sdk.microgrid._power_distributing._component_pool_status_tracker import ( ComponentPoolStatusTracker, ) -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 from frequenz.sdk.timeseries.battery_pool.messages import BatteryPoolReport from ...utils.component_data_streamer import MockComponentDataStreamer @@ -64,7 +64,7 @@ async def mocks(mocker: MockerFixture) -> typing.AsyncIterator[Mocks]: if microgrid._data_pipeline._DATA_PIPELINE is not None: microgrid._data_pipeline._DATA_PIPELINE = None await microgrid._data_pipeline.initialize( - ResamplerConfig(resampling_period=timedelta(seconds=0.1)) + ResamplerConfig2(resampling_period=timedelta(seconds=0.1)) ) streamer = MockComponentDataStreamer(mockgrid.mock_client) diff --git a/tests/timeseries/_ev_charger_pool/test_ev_charger_pool_control_methods.py b/tests/timeseries/_ev_charger_pool/test_ev_charger_pool_control_methods.py index 25de848b0..9c1837eab 100644 --- a/tests/timeseries/_ev_charger_pool/test_ev_charger_pool_control_methods.py +++ b/tests/timeseries/_ev_charger_pool/test_ev_charger_pool_control_methods.py @@ -23,7 +23,7 @@ from frequenz.sdk.microgrid._power_distributing._component_pool_status_tracker import ( ComponentPoolStatusTracker, ) -from frequenz.sdk.timeseries import ResamplerConfig, Sample3Phase +from frequenz.sdk.timeseries import ResamplerConfig2, Sample3Phase from frequenz.sdk.timeseries.ev_charger_pool import EVChargerPoolReport from ...microgrid.fixtures import _Mocks @@ -51,7 +51,7 @@ async def mocks(mocker: MockerFixture) -> typing.AsyncIterator[_Mocks]: if microgrid._data_pipeline._DATA_PIPELINE is not None: microgrid._data_pipeline._DATA_PIPELINE = None await microgrid._data_pipeline.initialize( - ResamplerConfig(resampling_period=timedelta(seconds=0.1)) + ResamplerConfig2(resampling_period=timedelta(seconds=0.1)) ) streamer = MockComponentDataStreamer(mockgrid.mock_client) diff --git a/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py b/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py index 10f25e7f3..db1cf0325 100644 --- a/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py +++ b/tests/timeseries/_pv_pool/test_pv_pool_control_methods.py @@ -19,7 +19,7 @@ from frequenz.sdk import microgrid from frequenz.sdk.microgrid import _power_distributing from frequenz.sdk.microgrid._data_pipeline import _DataPipeline -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 from frequenz.sdk.timeseries.pv_pool import PVPoolReport from ...microgrid.fixtures import _Mocks @@ -45,7 +45,7 @@ async def mocks(mocker: MockerFixture) -> typing.AsyncIterator[_Mocks]: if microgrid._data_pipeline._DATA_PIPELINE is not None: microgrid._data_pipeline._DATA_PIPELINE = None await microgrid._data_pipeline.initialize( - ResamplerConfig(resampling_period=timedelta(seconds=0.1)) + ResamplerConfig2(resampling_period=timedelta(seconds=0.1)) ) streamer = MockComponentDataStreamer(mockgrid.mock_client) diff --git a/tests/timeseries/mock_microgrid.py b/tests/timeseries/mock_microgrid.py index 326a4efa5..bf1e48c6d 100644 --- a/tests/timeseries/mock_microgrid.py +++ b/tests/timeseries/mock_microgrid.py @@ -29,7 +29,7 @@ from frequenz.sdk._internal._asyncio import cancel_and_await from frequenz.sdk.microgrid import _data_pipeline from frequenz.sdk.microgrid.component_graph import _MicrogridComponentGraph -from frequenz.sdk.timeseries import ResamplerConfig +from frequenz.sdk.timeseries import ResamplerConfig2 from ..utils import MockMicrogridClient from ..utils.component_data_wrapper import ( @@ -205,7 +205,7 @@ async def start(self, mocker: MockerFixture | None = None) -> None: self.init_mock_client(lambda mock_client: mock_client.initialize(local_mocker)) self.mock_resampler = MockResampler( mocker, - ResamplerConfig(timedelta(seconds=self._sample_rate_s)), + ResamplerConfig2(timedelta(seconds=self._sample_rate_s)), bat_inverter_ids=self.battery_inverter_ids, pv_inverter_ids=self.pv_inverter_ids, evc_ids=self.evc_ids, diff --git a/tests/timeseries/mock_resampler.py b/tests/timeseries/mock_resampler.py index ce4cc59b7..55d0a21a4 100644 --- a/tests/timeseries/mock_resampler.py +++ b/tests/timeseries/mock_resampler.py @@ -17,7 +17,7 @@ from frequenz.sdk._internal._asyncio import cancel_and_await from frequenz.sdk.microgrid._data_pipeline import _DataPipeline from frequenz.sdk.microgrid._data_sourcing import ComponentMetricRequest -from frequenz.sdk.timeseries import ResamplerConfig, Sample +from frequenz.sdk.timeseries import ResamplerConfig2, Sample from frequenz.sdk.timeseries.formula_engine._formula_generators._formula_generator import ( NON_EXISTING_COMPONENT_ID, ) @@ -31,7 +31,7 @@ class MockResampler: def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments self, mocker: MockerFixture, - resampler_config: ResamplerConfig, + resampler_config: ResamplerConfig2, bat_inverter_ids: list[ComponentId], pv_inverter_ids: list[ComponentId], evc_ids: list[ComponentId], From 57388f01689fbcfca6fae1265ff867bc69c79513 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 16:59:42 +0200 Subject: [PATCH 31/32] Test wall clock timer in resampled moving window tests These tests are very tricky and there seems to be some inconsistency between the monotonic and wall clock, the wall clock timer have less samples in the buffer at the end. This is probably some border condition because of the way we fake time. We also had to tune how to do wall clock time shifting to make sure the shift occurs before the timer wakes up. Finally we adjusted sample sending so it always sends an odd number of samples to make sure the resample is triggered at the points we expect it, otherwise again we could end up in border condition that were not the same for both timers. Signed-off-by: Leandro Lucarella --- tests/timeseries/test_moving_window.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/timeseries/test_moving_window.py b/tests/timeseries/test_moving_window.py index 35687cac5..6fce1e077 100644 --- a/tests/timeseries/test_moving_window.py +++ b/tests/timeseries/test_moving_window.py @@ -19,6 +19,7 @@ from frequenz.sdk.timeseries import ( MovingWindow, ResamplerConfig, + ResamplerConfig2, Sample, ) @@ -51,8 +52,12 @@ async def push_logical_meter_data( Sample(timestamp, Quantity(float(i)) if i is not None else None) ) if fake_time is not None: - await asyncio.sleep(1.0) - fake_time.shift(1) + # For the wall clock timer we need to make sure that the wall clock is + # adjusted just before the timer wakes up from sleeping, and then we need to + # make sure this function returns *after* the timer has woken up. + await asyncio.sleep(0.999) + fake_time.shift(1.0) + await asyncio.sleep(0.001) await asyncio.sleep(0.0) @@ -452,12 +457,13 @@ async def test_wait_for_samples() -> None: assert task.done() +@pytest.mark.parametrize("config_class", [ResamplerConfig, ResamplerConfig2]) async def test_wait_for_samples_with_resampling( - fake_time: time_machine.Coordinates, + config_class: type[ResamplerConfig], fake_time: time_machine.Coordinates ) -> None: """Test waiting for samples in a moving window with resampling.""" window, sender = init_moving_window( - timedelta(seconds=20), ResamplerConfig(resampling_period=timedelta(seconds=2)) + timedelta(seconds=20), config_class(resampling_period=timedelta(seconds=2)) ) async with window: task = asyncio.create_task(window.wait_for_samples(3)) @@ -469,7 +475,7 @@ async def test_wait_for_samples_with_resampling( task = asyncio.create_task(window.wait_for_samples(10)) await push_logical_meter_data( sender, - range(0, 11), + range(0, 10), fake_time=fake_time, start_ts=UNIX_EPOCH + timedelta(seconds=7), ) @@ -478,9 +484,9 @@ async def test_wait_for_samples_with_resampling( await push_logical_meter_data( sender, - range(0, 5), + range(0, 6), fake_time=fake_time, - start_ts=UNIX_EPOCH + timedelta(seconds=18), + start_ts=UNIX_EPOCH + timedelta(seconds=17), ) assert window.count_covered() == 10 assert not task.done() @@ -514,7 +520,11 @@ async def test_wait_for_samples_with_resampling( start_ts=UNIX_EPOCH + timedelta(seconds=39), ) assert window.count_covered() == 10 - assert window.count_valid() == 7 + # There is also an inconsistency here between the monotonic and wall clock, + # the wall clock timer have less samples in the buffer at the end. This is + # probably some border condition because of the way we fake time + is_wall_clock = config_class is ResamplerConfig2 + assert window.count_valid() == (5 if is_wall_clock else 7) assert task.done() From 88c6c538ca25a26d55fa0da190a0b44f610f3173 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Fri, 1 Aug 2025 16:59:56 +0200 Subject: [PATCH 32/32] Update release notes Signed-off-by: Leandro Lucarella --- RELEASE_NOTES.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2d494223b..42b8440c9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -12,8 +12,21 @@ ## New Features - +- A new configuration mode was added to the resampler (and thus the resampling actor and microgrid high-level interface). When passing a new `ResamplerConfig2` instance to the resampler, it will use a wall clock timer instead of a monotonic clock timer. This timer adjustes sleeps to account for drifts in the monotonic clock, and thus allows for more accurate resampling in cases where the monotonic clock drifts away from the wall clock. The monotonic clock timer option will be deprecated in the future, as it is not really suitable for resampling. The new `ResamplerConfig2` class accepts a `WallClockTimerConfig` to fine-tune the wall clock timer behavior, if necessary. + + Example usage: + + ```python + from frequenz.sdk import microgrid + from frequenz.sdk.timeseries import ResamplerConfig2 + + await microgrid.initialize( + MICROGRID_API_URL, + # Just replace the old `ResamplerConfig` with the new `ResamplerConfig2` + resampler_config=ResamplerConfig2(resampling_period=timedelta(seconds=1.0)), + ) + ``` ## Bug Fixes - +- When using the new wall clock timer in the resampmler, it will now resync to the system time if it drifts away for more than a resample period, and do dynamic adjustments to the timer if the monotonic clock has a small drift compared to the wall clock.