From 4232ababaf9cdfd6fa68746320cf53705b8c99b5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Mar 2023 10:55:31 -1000 Subject: [PATCH] fix slow tests --- homeassistant/components/recorder/core.py | 20 ++++++++- homeassistant/components/recorder/purge.py | 3 ++ homeassistant/components/recorder/queries.py | 7 ++++ tests/components/recorder/db_schema_28.py | 43 ++++++++++++++++++++ tests/components/recorder/db_schema_30.py | 3 ++ 5 files changed, 74 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/recorder/core.py b/homeassistant/components/recorder/core.py index a38ffd8b3b7..97d72c7f85c 100644 --- a/homeassistant/components/recorder/core.py +++ b/homeassistant/components/recorder/core.py @@ -82,6 +82,7 @@ from .queries import ( find_shared_data_id, get_shared_attributes, get_shared_event_datas, + has_event_type_to_migrate, ) from .run_history import RunHistory from .table_managers.event_types import EventTypeManager @@ -693,11 +694,26 @@ class Recorder(threading.Thread): _LOGGER.debug("Recorder processing the queue") self._adjust_lru_size() self.hass.add_job(self._async_set_recorder_ready_migration_done) - self.queue_task(ContextIDMigrationTask()) - self.queue_task(EventTypeIDMigrationTask()) + self._activate_table_managers_or_migrate() self._run_event_loop() self._shutdown() + def _activate_table_managers_or_migrate(self) -> None: + """Activate the table managers or schedule migrations.""" + # Currently we always check if context ids need to be migrated + # since there are multiple tables. This could be optimized + # to check both the states and events table to see if there + # are any missing and avoid inserting the task but it currently + # is not needed since there is no dependent code branching + # on the result of the migration. + self.queue_task(ContextIDMigrationTask()) + with session_scope(session=self.get_session()) as session: + if session.execute(has_event_type_to_migrate()).scalar(): + self.queue_task(EventTypeIDMigrationTask()) + else: + _LOGGER.debug("Activating event type manager as all data is migrated") + self.event_type_manager.active = True + def _run_event_loop(self) -> None: """Run the event loop for the recorder.""" # Use a session for the event read loop diff --git a/homeassistant/components/recorder/purge.py b/homeassistant/components/recorder/purge.py index 5b39867ffbc..a02b34fe529 100644 --- a/homeassistant/components/recorder/purge.py +++ b/homeassistant/components/recorder/purge.py @@ -577,6 +577,9 @@ def _purge_old_event_types(instance: Recorder, session: Session) -> None: purged_event_types.add(event_type) event_type_ids.add(event_type_id) + if not event_type_ids: + return + deleted_rows = session.execute(delete_event_types_rows(event_type_ids)) _LOGGER.debug("Deleted %s event types", deleted_rows) diff --git a/homeassistant/components/recorder/queries.py b/homeassistant/components/recorder/queries.py index 5a558fc6208..ddc04d820a1 100644 --- a/homeassistant/components/recorder/queries.py +++ b/homeassistant/components/recorder/queries.py @@ -716,6 +716,13 @@ def find_event_type_to_migrate() -> StatementLambdaElement: ) +def has_event_type_to_migrate() -> StatementLambdaElement: + """Check if there are event_types to migrate.""" + return lambda_stmt( + lambda: select(Events.event_id).filter(Events.event_type_id.is_(None)).limit(1) + ) + + def find_states_context_ids_to_migrate() -> StatementLambdaElement: """Find events context_ids to migrate.""" return lambda_stmt( diff --git a/tests/components/recorder/db_schema_28.py b/tests/components/recorder/db_schema_28.py index 422f317a6f1..f7152cec508 100644 --- a/tests/components/recorder/db_schema_28.py +++ b/tests/components/recorder/db_schema_28.py @@ -21,6 +21,7 @@ from sqlalchemy import ( Identity, Index, Integer, + LargeBinary, SmallInteger, String, Text, @@ -54,6 +55,7 @@ DB_TIMEZONE = "+00:00" TABLE_EVENTS = "events" TABLE_EVENT_DATA = "event_data" +TABLE_EVENT_TYPES = "event_types" TABLE_STATES = "states" TABLE_STATE_ATTRIBUTES = "state_attributes" TABLE_RECORDER_RUNS = "recorder_runs" @@ -68,6 +70,7 @@ ALL_TABLES = [ TABLE_STATE_ATTRIBUTES, TABLE_EVENTS, TABLE_EVENT_DATA, + TABLE_EVENT_TYPES, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES, TABLE_STATISTICS, @@ -98,6 +101,11 @@ DOUBLE_TYPE = ( ) EVENT_ORIGIN_ORDER = [EventOrigin.local, EventOrigin.remote] EVENT_ORIGIN_TO_IDX = {origin: idx for idx, origin in enumerate(EVENT_ORIGIN_ORDER)} +CONTEXT_ID_BIN_MAX_LENGTH = 16 +EVENTS_CONTEXT_ID_BIN_INDEX = "ix_events_context_id_bin" +STATES_CONTEXT_ID_BIN_INDEX = "ix_states_context_id_bin" + +TIMESTAMP_TYPE = DOUBLE_TYPE class Events(Base): # type: ignore[misc,valid-type] @@ -107,6 +115,12 @@ class Events(Base): # type: ignore[misc,valid-type] # Used for fetching events at a specific time # see logbook Index("ix_events_event_type_time_fired", "event_type", "time_fired"), + Index( + EVENTS_CONTEXT_ID_BIN_INDEX, + "context_id_bin", + mysql_length=CONTEXT_ID_BIN_MAX_LENGTH, + mariadb_length=CONTEXT_ID_BIN_MAX_LENGTH, + ), {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, ) __tablename__ = TABLE_EVENTS @@ -116,11 +130,27 @@ class Events(Base): # type: ignore[misc,valid-type] origin = Column(String(MAX_LENGTH_EVENT_ORIGIN)) # no longer used origin_idx = Column(SmallInteger) time_fired = Column(DATETIME_TYPE, index=True) + time_fired_ts = Column( + TIMESTAMP_TYPE, index=True + ) # *** Not originally in v30, only added for recorder to startup ok context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True) context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) data_id = Column(Integer, ForeignKey("event_data.data_id"), index=True) + context_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v28, only added for recorder to startup ok + context_user_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v28, only added for recorder to startup ok + context_parent_id_bin = Column( + LargeBinary(CONTEXT_ID_BIN_MAX_LENGTH) + ) # *** Not originally in v28, only added for recorder to startup ok + event_type_id = Column( + Integer, ForeignKey("event_types.event_type_id"), index=True + ) # *** Not originally in v28, only added for recorder to startup ok event_data_rel = relationship("EventData") + event_type_rel = relationship("EventTypes") def __repr__(self) -> str: """Return string representation of instance for debugging.""" @@ -214,6 +244,19 @@ class EventData(Base): # type: ignore[misc,valid-type] return {} +# *** Not originally in v28, only added for recorder to startup ok +# This is not being tested by the v28 statistics migration tests +class EventTypes(Base): # type: ignore[misc,valid-type] + """Event type history.""" + + __table_args__ = ( + {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"}, + ) + __tablename__ = TABLE_EVENT_TYPES + event_type_id = Column(Integer, Identity(), primary_key=True) + event_type = Column(String(MAX_LENGTH_EVENT_EVENT_TYPE)) + + class States(Base): # type: ignore[misc,valid-type] """State change history.""" diff --git a/tests/components/recorder/db_schema_30.py b/tests/components/recorder/db_schema_30.py index c219d71011e..ed9fb89e464 100644 --- a/tests/components/recorder/db_schema_30.py +++ b/tests/components/recorder/db_schema_30.py @@ -214,6 +214,9 @@ class Events(Base): # type: ignore[misc,valid-type] origin = Column(String(MAX_LENGTH_EVENT_ORIGIN)) # no longer used for new rows origin_idx = Column(SmallInteger) time_fired = Column(DATETIME_TYPE, index=True) + time_fired_ts = Column( + TIMESTAMP_TYPE, index=True + ) # *** Not originally in v30, only added for recorder to startup ok context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True) context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID)) context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))