Force a real compile in the template compile benchmark

.core_files.yaml

@@ -11,6 +11,12 @@ core: &core
   - requirements.txt
   - setup.cfg
 
+# Performance benchmark suite (CodSpeed). Only gates the benchmark job; kept out
+# of the `any` aggregate below so it does not pull in the full test suite.
+benchmarks: &benchmarks
+  - benchmarks/**
+  - requirements_test.txt
+
 # Our base platforms, that are used by other integrations
 base_platforms: &base_platforms
   - homeassistant/components/ai_task/**

.github/workflows/ci.yaml

@@ -812,6 +812,55 @@ jobs:
           python --version
           mypy --num-workers=4 $(printf "homeassistant/components/%s " ${INTEGRATIONS_GLOB})
 
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write # OIDC token CodSpeed mints (no CODSPEED_TOKEN secret)
+    needs:
+      - info
+      - base
+    # Run only when core code or the benchmark suite itself changed. Skipped on
+    # forks, where the OIDC token CodSpeed needs is unavailable. Pushes to dev
+    # that touch core refresh the CodSpeed baseline.
+    if: >-
+      needs.info.outputs.lint_only != 'true'
+      && (github.event_name != 'pull_request'
+        || !github.event.pull_request.head.repo.fork)
+      && (contains(fromJSON(needs.info.outputs.core), 'core')
+        || contains(fromJSON(needs.info.outputs.core), 'benchmarks'))
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Set up Python ${{ fromJson(needs.info.outputs.python_versions)[0] }}
+        id: python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: ${{ fromJson(needs.info.outputs.python_versions)[0] }}
+          check-latest: true
+      - name: Restore full Python virtual environment
+        id: cache-venv
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: venv
+          fail-on-cache-miss: true
+          key: >-
+            ${{ runner.os }}-${{ runner.arch }}-${{ steps.python.outputs.python-version }}-${{
+            needs.info.outputs.python_cache_key }}
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@a4a36bb07c0638b0b4ca52bf1f3dad1b4289e52f # v4.18.1
+        with:
+          # v4 makes `mode` required; "simulation" is the instrumented
+          # measurement that gives stable, machine-independent results.
+          mode: simulation
+          # No token: auth uses the OIDC id-token minted by the job permissions.
+          run: |
+            . venv/bin/activate
+            pytest benchmarks --codspeed --no-cov -o addopts=""
+
   prepare-pytest-full:
     name: Split tests for full run
     runs-on: ubuntu-24.04

benchmarks/__init__.py

@@ -0,0 +1 @@
+"""CodSpeed performance benchmarks for Home Assistant core hot paths."""

benchmarks/conftest.py

@@ -0,0 +1,45 @@
+"""Shared fixtures for the CodSpeed benchmark suite.
+
+These benchmarks live outside ``tests`` on purpose: ``testpaths`` only points at
+``tests``, so the regular suite never collects them. CodSpeed runs them with
+``pytest benchmarks --codspeed`` and tracks the results per pull request.
+"""
+
+from collections.abc import AsyncGenerator, Callable
+
+import pytest
+
+from homeassistant.core import HomeAssistant
+from tests.common import async_test_home_assistant
+
+
+@pytest.fixture
+async def hass() -> AsyncGenerator[HomeAssistant]:
+    """Return a running Home Assistant instance for benchmarking.
+
+    Most hot paths under test (``async_fire``, ``async_set``, ``async_render``)
+    are ``@callback`` methods, so the benchmark fixture can drive them
+    synchronously from within the running loop.
+    """
+    async with async_test_home_assistant() as hass:
+        yield hass
+
+
+@pytest.fixture
+def populate_states(hass: HomeAssistant) -> Callable[[int], None]:
+    """Return a helper that fills the state machine with ``count`` sensors.
+
+    Used by the scaling benchmarks to measure a path at several sizes, so an
+    algorithmic regression shows up as the curve bending instead of hiding
+    behind a single constant-factor number.
+    """
+
+    def _populate(count: int) -> None:
+        for index in range(count):
+            hass.states.async_set(
+                f"sensor.bench_{index}",
+                str(index),
+                {"friendly_name": f"Bench {index}", "unit_of_measurement": "W"},
+            )
+
+    return _populate

benchmarks/test_core_events.py

@@ -0,0 +1,148 @@
+"""CodSpeed benchmarks for the event bus and event helpers.
+
+The event bus carries every state change, and ``async_track_state_change_event``
+is the routing layer almost every automation, template and trigger sits on. A
+regression in either is felt across the whole system.
+
+Run locally with: ``pytest benchmarks --codspeed``.
+"""
+
+from collections.abc import Callable
+
+import pytest
+from pytest_codspeed import BenchmarkFixture
+
+from homeassistant.core import Event, HomeAssistant, callback
+from homeassistant.helpers.dispatcher import (
+    async_dispatcher_connect,
+    async_dispatcher_send,
+)
+from homeassistant.helpers.event import async_call_later, async_track_state_change_event
+
+
+@callback
+def _noop(event: Event) -> None:
+    """Do nothing, cheaply."""
+
+
+def test_event_fire_no_listeners(
+    benchmark: BenchmarkFixture, hass: HomeAssistant
+) -> None:
+    """Fire an event nobody listens to (the bare dispatch cost)."""
+    benchmark(lambda: hass.bus.async_fire("benchmark_event", {"value": 1}))
+
+
+@pytest.mark.parametrize("listeners", [1, 10])
+def test_event_fire_callbacks(
+    benchmark: BenchmarkFixture, hass: HomeAssistant, listeners: int
+) -> None:
+    """Fire an event with N callback listeners that run inline."""
+    fired = 0
+
+    @callback
+    def listener(event: Event) -> None:
+        nonlocal fired
+        fired += 1
+
+    for _ in range(listeners):
+        hass.bus.async_listen("benchmark_event", listener)
+
+    benchmark(lambda: hass.bus.async_fire("benchmark_event", {"value": 1}))
+
+    assert fired
+
+
+def test_event_fire_filtered_reject(
+    benchmark: BenchmarkFixture, hass: HomeAssistant
+) -> None:
+    """Fire an event whose listener is gated out by an event_filter.
+
+    The filter runs but the listener does not, so this isolates the filter
+    short-circuit cost from the listener body.
+    """
+
+    @callback
+    def event_filter(event_data: dict) -> bool:
+        return False
+
+    hass.bus.async_listen("benchmark_event", _noop, event_filter=event_filter)
+
+    benchmark(lambda: hass.bus.async_fire("benchmark_event", {"value": 1}))
+
+
+def test_state_change_tracked(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Fire a state change routed to a tracked entity's listener.
+
+    This is the real automation hot path: ``async_set`` fires
+    EVENT_STATE_CHANGED, the dispatcher does a dict lookup on the entity_id and
+    runs the inline callback.
+    """
+    fired = 0
+
+    @callback
+    def listener(event: Event) -> None:
+        nonlocal fired
+        fired += 1
+
+    async_track_state_change_event(hass, "sensor.tracked", listener)
+    counter = 0
+
+    def _set() -> None:
+        nonlocal counter
+        counter += 1
+        hass.states.async_set("sensor.tracked", str(counter))
+
+    benchmark(_set)
+
+    assert fired
+
+
+def test_state_change_untracked(
+    benchmark: BenchmarkFixture, hass: HomeAssistant
+) -> None:
+    """Fire a state change for an entity nobody tracks (the dict-miss path).
+
+    Tracking is installed for a different entity, so the dispatcher's lookup
+    misses and returns fast. This is the common case on a busy bus.
+    """
+    async_track_state_change_event(hass, "sensor.tracked", _noop)
+    counter = 0
+
+    def _set() -> None:
+        nonlocal counter
+        counter += 1
+        hass.states.async_set("sensor.untracked", str(counter))
+
+    benchmark(_set)
+
+
+@pytest.mark.parametrize("receivers", [0, 1, 10])
+def test_dispatcher_send(
+    benchmark: BenchmarkFixture, hass: HomeAssistant, receivers: int
+) -> None:
+    """Send a dispatcher signal to N connected receivers."""
+    fired = 0
+
+    @callback
+    def receiver(*args: object) -> None:
+        nonlocal fired
+        fired += 1
+
+    for _ in range(receivers):
+        async_dispatcher_connect(hass, "benchmark_signal", receiver)
+
+    benchmark(lambda: async_dispatcher_send(hass, "benchmark_signal", 1))
+
+
+def test_call_later_schedule(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Schedule a delayed callback and cancel it (the timer-tracking cost).
+
+    Cancelling inside the measured call keeps timers from piling up on the loop
+    across iterations.
+    """
+
+    def _schedule() -> None:
+        cancel: Callable[[], None] = async_call_later(hass, 60, _noop)
+        cancel()
+
+    benchmark(_schedule)

benchmarks/test_core_states.py

@@ -0,0 +1,138 @@
+"""CodSpeed benchmarks for the state machine and entity write path.
+
+Every state update in Home Assistant flows through ``StateMachine.async_set``,
+and every entity that pushes an update lands in ``Entity._async_write_ha_state``.
+These are among the busiest call sites in the whole process.
+
+Run locally with: ``pytest benchmarks --codspeed``.
+"""
+
+from collections.abc import Callable
+from typing import Any
+
+import pytest
+from pytest_codspeed import BenchmarkFixture
+
+from homeassistant.core import HomeAssistant, State
+from homeassistant.helpers.entity import Entity
+
+_ATTRS = {
+    "friendly_name": "Benchmark light",
+    "brightness": 255,
+    "color_temp_kelvin": 4000,
+    "supported_color_modes": ["color_temp", "rgb"],
+}
+
+
+def test_state_set_create(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Write a brand new entity into the state machine (the create branch).
+
+    ``pedantic`` with a teardown that removes the entity keeps every measured
+    call on the create path; without it only the first call creates and the rest
+    measure the update path.
+    """
+    benchmark.pedantic(
+        lambda: hass.states.async_set("light.benchmark", "on", _ATTRS),
+        teardown=lambda: hass.states.async_remove("light.benchmark"),
+        rounds=1000,
+    )
+
+
+def test_state_set_update(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Write a changed state for an existing entity (the update branch)."""
+    counter = 0
+
+    def _set() -> None:
+        nonlocal counter
+        counter += 1
+        hass.states.async_set("light.benchmark", str(counter), _ATTRS)
+
+    benchmark(_set)
+
+    assert hass.states.get("light.benchmark") is not None
+
+
+def test_state_set_report(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Re-set an unchanged state (the EVENT_STATE_REPORTED fast path).
+
+    Polling integrations hammer this branch: same value, same attributes, over
+    and over. It fires a lightweight reported event instead of a state change.
+    """
+    hass.states.async_set("sensor.benchmark", "21.5", _ATTRS)
+
+    benchmark(lambda: hass.states.async_set("sensor.benchmark", "21.5", _ATTRS))
+
+
+def test_state_get(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Look a single state up by entity_id."""
+    hass.states.async_set("sensor.benchmark", "21.5", _ATTRS)
+
+    state: State | None = benchmark(lambda: hass.states.get("sensor.benchmark"))
+
+    assert state is not None
+
+
+@pytest.mark.parametrize("count", [10, 100, 1000])
+def test_state_all(
+    benchmark: BenchmarkFixture,
+    populate_states: Callable[[int], None],
+    hass: HomeAssistant,
+    count: int,
+) -> None:
+    """Read the full state list out of a populated machine, at several sizes."""
+    populate_states(count)
+
+    states: list[State] = benchmark(hass.states.async_all)
+
+    assert len(states) == count
+
+
+@pytest.mark.parametrize("count", [10, 100, 1000])
+def test_state_entity_ids(
+    benchmark: BenchmarkFixture,
+    populate_states: Callable[[int], None],
+    hass: HomeAssistant,
+    count: int,
+) -> None:
+    """List entity ids out of a populated machine, at several sizes."""
+    populate_states(count)
+
+    entity_ids = benchmark(hass.states.async_entity_ids)
+
+    assert len(entity_ids) == count
+
+
+class _BenchmarkEntity(Entity):
+    """A minimal entity carrying capability and extra state attributes."""
+
+    _attr_should_poll = False
+    _attr_name = "Benchmark"
+    _attr_supported_features = 3
+
+    def __init__(self, state: str) -> None:
+        """Initialize the benchmark entity."""
+        self._attr_state = state
+        self._attr_extra_state_attributes: dict[str, Any] = {
+            "brightness": 255,
+            "color_temp_kelvin": 4000,
+        }
+
+    @property
+    def capability_attributes(self) -> dict[str, Any]:
+        """Return capability attributes, assembled on every write."""
+        return {"supported_color_modes": ["color_temp", "rgb"]}
+
+
+def test_entity_write(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Write an entity's state through the entity layer.
+
+    This measures ``__async_calculate_state`` (assembling state and attributes
+    from the entity's properties) plus the ``async_set`` it lands in.
+    """
+    entity = _BenchmarkEntity("on")
+    entity.hass = hass
+    entity.entity_id = "light.benchmark"
+
+    benchmark(entity._async_write_ha_state)  # noqa: SLF001
+
+    assert hass.states.get("light.benchmark") is not None

benchmarks/test_core_template.py

@@ -0,0 +1,112 @@
+"""CodSpeed benchmarks for the template engine.
+
+Templates render on dashboards, in automations and in many entity attributes.
+Both the compile step and the warm render matter, and templates that walk the
+state machine scale with the number of entities.
+
+Run locally with: ``pytest benchmarks --codspeed``.
+"""
+
+from collections.abc import Callable
+
+import pytest
+from pytest_codspeed import BenchmarkFixture
+
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers.template import Template
+
+_STATES_TEMPLATE = (
+    "{{ (states('sensor.power') | float / 1000) | round(2) }} kW "
+    "{{ is_state('binary_sensor.motion', 'on') }} "
+    "{{ state_attr('sensor.power', 'unit_of_measurement') }}"
+)
+
+
+def test_template_compile(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None:
+    """Compile a template from source (parse plus codegen).
+
+    A unique source on every call (a varying Jinja comment) keeps the
+    environment's template cache missing, so each run actually compiles instead
+    of returning cached bytecode. The comment is stripped during compilation, so
+    the cost matches the real template.
+    """
+    counter = 0
+
+    def _compile() -> Template:
+        nonlocal counter
+        counter += 1
+        template = Template(f"{{# {counter} #}}{_STATES_TEMPLATE}", hass)
+        template.ensure_valid()
+        return template
+
+    template = benchmark(_compile)
+
+    assert template.is_static is False
+
+
+def test_template_render_simple(
+    benchmark: BenchmarkFixture, hass: HomeAssistant
+) -> None:
+    """Render a pure-math template (warm), the engine's baseline overhead."""
+    template = Template("{{ 1 + 1 }}", hass)
+    template.ensure_valid()
+
+    result = benchmark(template.async_render)
+
+    assert result == 2
+
+
+def test_template_render_states(
+    benchmark: BenchmarkFixture, hass: HomeAssistant
+) -> None:
+    """Render a template that reads states, attributes and a filter (warm)."""
+    hass.states.async_set("sensor.power", "1200", {"unit_of_measurement": "W"})
+    hass.states.async_set("binary_sensor.motion", "on")
+    template = Template(_STATES_TEMPLATE, hass)
+    template.ensure_valid()
+
+    result = benchmark(template.async_render)
+
+    assert result.startswith("1.2 kW")
+
+
+def test_template_render_to_info(
+    benchmark: BenchmarkFixture, hass: HomeAssistant
+) -> None:
+    """Render and collect the entity dependency filter (the tracking path).
+
+    ``async_render_to_info`` is what template triggers and template entities use
+    to learn which entities to subscribe to.
+    """
+    hass.states.async_set("sensor.power", "1200", {"unit_of_measurement": "W"})
+    hass.states.async_set("binary_sensor.motion", "on")
+    template = Template(_STATES_TEMPLATE, hass)
+    template.ensure_valid()
+
+    info = benchmark(template.async_render_to_info)
+
+    assert info.entities or info.all_states
+
+
+@pytest.mark.parametrize("count", [10, 100, 1000])
+def test_template_iterate_states(
+    benchmark: BenchmarkFixture,
+    populate_states: Callable[[int], None],
+    hass: HomeAssistant,
+    count: int,
+) -> None:
+    """Render a template that walks every sensor state, at several sizes.
+
+    This is where an O(n) template touches an O(n) state machine; the cost
+    should grow linearly and a worse-than-linear regression should stand out.
+    """
+    populate_states(count)
+    template = Template(
+        "{{ states.sensor | selectattr('state', 'eq', '1') | list | count }}",
+        hass,
+    )
+    template.ensure_valid()
+
+    result = benchmark(template.async_render)
+
+    assert result == 1

pyproject.toml

@@ -437,6 +437,13 @@ runtime-typing = false
 [tool.pylint.CODE_STYLE]
 max-line-length-suggestions = 72
 
+[tool.uv]
+# surepy (a surepetcare dependency) carries a stale upper pin on rich (<11) that
+# blocks pytest-codspeed (needs rich>=13.8.1) from resolving in the same
+# environment. rich keeps the Console API surepy uses backwards compatible, so
+# override the pin rather than holding the benchmark tooling back.
+override-dependencies = ["rich>=13.8.1"]
+
 [tool.pytest.ini_options]
 pythonpath = ["pylint/plugins"]
 testpaths = ["tests"]
@@ -877,6 +884,9 @@ split-on-trailing-comma = false
 "homeassistant/components/*/*/*" = ["TID252"]
 "tests/components/*/*/*" = ["TID252"]
 
+# Benchmarks reuse the test helpers to spin up a Home Assistant instance
+"benchmarks/**" = ["TID251"]
+
 # Temporary
 "homeassistant/**" = ["PTH"]
 "tests/**" = ["PTH"]

requirements_test.txt

@@ -25,6 +25,7 @@ pylint-per-file-ignores==3.2.1
 pipdeptree==2.26.1
 pytest-asyncio==1.4.0
 pytest-aiohttp==1.1.1
+pytest-codspeed==5.0.3
 pytest-cov==7.1.0
 pytest-freezer==0.4.9
 pytest-github-actions-annotate-failures==0.4.2