From 9afabaaba153af4f3d453da12ed434f5ca6dc206 Mon Sep 17 00:00:00 2001 From: Erik Montnemery Date: Mon, 7 Sep 2026 09:38:44 +0200 Subject: [PATCH] Trim cached orjson fragments kept by registries (#181240) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- homeassistant/helpers/area_registry.py | 32 +++-- homeassistant/helpers/device_registry.py | 144 +++++++++++----------- homeassistant/helpers/entity_registry.py | 148 ++++++++++++----------- homeassistant/helpers/json.py | 22 ++++ tests/helpers/test_json.py | 70 +++++++++++ 5 files changed, 254 insertions(+), 162 deletions(-) diff --git a/homeassistant/helpers/area_registry.py b/homeassistant/helpers/area_registry.py index 1835fb6af80f..6da7608fcf36 100644 --- a/homeassistant/helpers/area_registry.py +++ b/homeassistant/helpers/area_registry.py @@ -14,7 +14,7 @@ from homeassistant.util.event_type import EventType from homeassistant.util.hass_dict import HassKey from . import device_registry as dr -from .json import json_bytes, json_fragment +from .json import cached_json_fragment, json_fragment from .normalized_name_base_registry import ( NormalizedNameBaseRegistryEntry, NormalizedNameBaseRegistryItems, @@ -87,22 +87,20 @@ class AreaEntry(NormalizedNameBaseRegistryEntry): @under_cached_property def json_fragment(self) -> json_fragment: """Return a JSON representation of this AreaEntry.""" - return json_fragment( - json_bytes( - { - "aliases": list(self.aliases), - "area_id": self.id, - "floor_id": self.floor_id, - "humidity_entity_id": self.humidity_entity_id, - "icon": self.icon, - "labels": list(self.labels), - "name": self.name, - "picture": self.picture, - "temperature_entity_id": self.temperature_entity_id, - "created_at": self.created_at.timestamp(), - "modified_at": self.modified_at.timestamp(), - } - ) + return cached_json_fragment( + { + "aliases": list(self.aliases), + "area_id": self.id, + "floor_id": self.floor_id, + "humidity_entity_id": self.humidity_entity_id, + "icon": self.icon, + "labels": list(self.labels), + "name": self.name, + "picture": self.picture, + "temperature_entity_id": self.temperature_entity_id, + "created_at": self.created_at.timestamp(), + "modified_at": self.modified_at.timestamp(), + } ) diff --git a/homeassistant/helpers/device_registry.py b/homeassistant/helpers/device_registry.py index e08aec135e6d..a0974e7f1b27 100644 --- a/homeassistant/helpers/device_registry.py +++ b/homeassistant/helpers/device_registry.py @@ -52,7 +52,13 @@ from .frame import ( get_integration_frame, report_usage, ) -from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes, json_fragment +from .json import ( + JSON_DUMP, + cached_json_bytes, + cached_json_fragment, + find_paths_unserializable_data, + json_fragment, +) from .registry import BaseRegistry, BaseRegistryItems, RegistryIndexType from .typing import UNDEFINED, UndefinedType @@ -435,7 +441,7 @@ class BaseDeviceEntry: """Return a cached JSON representation of the entry.""" try: dict_repr = self.dict_repr - return json_bytes(dict_repr) + return cached_json_bytes(dict_repr) except ValueError, TypeError: _LOGGER.error( "Unable to serialize entry %s to JSON. Bad data found at %s", @@ -564,39 +570,35 @@ class DeviceEntry(BaseDeviceEntry): @under_cached_property def as_storage_fragment(self) -> json_fragment: """Return a json fragment for storage.""" - return json_fragment( - json_bytes( - { - "area_id": self.area_id, - "config_entry_id": self.config_entry_id, - "config_subentry_id": self.config_subentry_id, - "configuration_url": self.configuration_url, - "connections": list(self.connections), - "created_at": self.created_at, - "disabled_by": self.disabled_by, - "entry_type": self.entry_type, - "hw_version": self.hw_version, - "id": self.id, - "identifiers": list(self.identifiers), - "labels": list(self.labels), - "composite_device_id": self.composite_device_id, - "composite_primary_config_entry": ( - self.composite_primary_config_entry - ), - "split_at": self.split_at, - "manufacturer": self.manufacturer, - "model": self.model, - "model_id": self.model_id, - "modified_at": self.modified_at, - "name_by_user": self.name_by_user, - "name": self.name, - "has_composite_identifiers": (self.has_composite_identifiers), - "primary_config_entry": self.primary_config_entry, - "serial_number": self.serial_number, - "sw_version": self.sw_version, - "via_device_id": self.via_device_id, - } - ) + return cached_json_fragment( + { + "area_id": self.area_id, + "config_entry_id": self.config_entry_id, + "config_subentry_id": self.config_subentry_id, + "configuration_url": self.configuration_url, + "connections": list(self.connections), + "created_at": self.created_at, + "disabled_by": self.disabled_by, + "entry_type": self.entry_type, + "hw_version": self.hw_version, + "id": self.id, + "identifiers": list(self.identifiers), + "labels": list(self.labels), + "composite_device_id": self.composite_device_id, + "composite_primary_config_entry": self.composite_primary_config_entry, + "split_at": self.split_at, + "manufacturer": self.manufacturer, + "model": self.model, + "model_id": self.model_id, + "modified_at": self.modified_at, + "name_by_user": self.name_by_user, + "name": self.name, + "has_composite_identifiers": (self.has_composite_identifiers), + "primary_config_entry": self.primary_config_entry, + "serial_number": self.serial_number, + "sw_version": self.sw_version, + "via_device_id": self.via_device_id, + } ) @property @@ -686,23 +688,21 @@ class ChildDeviceEntry(BaseDeviceEntry): @under_cached_property def as_storage_fragment(self) -> json_fragment: """Return a json fragment for storage.""" - return json_fragment( - json_bytes( - { - "area_id": self.area_id, - "config_entry_id": self.config_entry_id, - "config_subentry_id": self.config_subentry_id, - "created_at": self.created_at, - "disabled_by": self.disabled_by, - "id": self.id, - "identifiers": list(self.identifiers), - "labels": list(self.labels), - "modified_at": self.modified_at, - "name_by_user": self.name_by_user, - "name": self.name, - "parent_device_id": self.parent_device_id, - } - ) + return cached_json_fragment( + { + "area_id": self.area_id, + "config_entry_id": self.config_entry_id, + "config_subentry_id": self.config_subentry_id, + "created_at": self.created_at, + "disabled_by": self.disabled_by, + "id": self.id, + "identifiers": list(self.identifiers), + "labels": list(self.labels), + "modified_at": self.modified_at, + "name_by_user": self.name_by_user, + "name": self.name, + "parent_device_id": self.parent_device_id, + } ) @@ -849,27 +849,25 @@ class DeletedDeviceEntry: @under_cached_property def as_storage_fragment(self) -> json_fragment: """Return a json fragment for storage.""" - return json_fragment( - json_bytes( - { - "area_id": self.area_id, - "config_entry_id": self.config_entry_id, - "config_subentry_id": self.config_subentry_id, - "connections": list(self.connections), - "created_at": self.created_at, - "disabled_by": self.disabled_by - if self.disabled_by is not UNDEFINED - else None, - "disabled_by_undefined": self.disabled_by is UNDEFINED, - "identifiers": list(self.identifiers), - "id": self.id, - "labels": list(self.labels), - "modified_at": self.modified_at, - "name_by_user": self.name_by_user, - "orphaned_timestamp": self.orphaned_timestamp, - "domain": self.domain, - } - ) + return cached_json_fragment( + { + "area_id": self.area_id, + "config_entry_id": self.config_entry_id, + "config_subentry_id": self.config_subentry_id, + "connections": list(self.connections), + "created_at": self.created_at, + "disabled_by": self.disabled_by + if self.disabled_by is not UNDEFINED + else None, + "disabled_by_undefined": self.disabled_by is UNDEFINED, + "identifiers": list(self.identifiers), + "id": self.id, + "labels": list(self.labels), + "modified_at": self.modified_at, + "name_by_user": self.name_by_user, + "orphaned_timestamp": self.orphaned_timestamp, + "domain": self.domain, + } ) diff --git a/homeassistant/helpers/entity_registry.py b/homeassistant/helpers/entity_registry.py index 1d9f4dce8341..a2a2ce1ca295 100644 --- a/homeassistant/helpers/entity_registry.py +++ b/homeassistant/helpers/entity_registry.py @@ -53,7 +53,13 @@ from .device_registry import ( EventDeviceRegistryUpdatedData, ) from .frame import ReportBehavior, report_usage -from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes, json_fragment +from .json import ( + JSON_DUMP, + cached_json_bytes, + cached_json_fragment, + find_paths_unserializable_data, + json_fragment, +) from .registry import BaseRegistry, BaseRegistryItems, RegistryIndexType from .singleton import singleton from .typing import UNDEFINED, UndefinedType @@ -319,7 +325,9 @@ class RegistryEntry: """ try: dict_repr = self._as_display_dict - json_repr: bytes | None = json_bytes(dict_repr) if dict_repr else None + json_repr: bytes | None = ( + cached_json_bytes(dict_repr) if dict_repr else None + ) except ValueError, TypeError: _LOGGER.error( "Unable to serialize entry %s to JSON. Bad data found at %s", @@ -386,7 +394,7 @@ class RegistryEntry: """Return a cached partial JSON representation of the entry.""" try: dict_repr = self.as_partial_dict - return json_bytes(dict_repr) + return cached_json_bytes(dict_repr) except ValueError, TypeError: _LOGGER.error( "Unable to serialize entry %s to JSON. Bad data found at %s", @@ -400,43 +408,41 @@ class RegistryEntry: @under_cached_property def as_storage_fragment(self) -> json_fragment: """Return a json fragment for storage.""" - return json_fragment( - json_bytes( - { - "aliases": self.compat_aliases, - "aliases_v2": _serialize_aliases(self.aliases), - "area_id": self.area_id, - "categories": self.categories, - "capabilities": self.capabilities, - "config_entry_id": self.config_entry_id, - "config_subentry_id": self.config_subentry_id, - "created_at": self.created_at, - "device_class": self.device_class, - "device_id": self.device_id, - "disabled_by": self.disabled_by, - "entity_category": self.entity_category, - "entity_id": self.entity_id, - "hidden_by": self.hidden_by, - "icon": self.icon, - "id": self.id, - "has_entity_name": self.has_entity_name, - "labels": list(self.labels), - "modified_at": self.modified_at, - "name": self.name, - "object_id_base": self.object_id_base, - "options": self.options, - "original_device_class": self.original_device_class, - "original_icon": self.original_icon, - "original_name": self.original_name, - "platform": self.platform, - "suggested_object_id": self.suggested_object_id, - "supported_features": self.supported_features, - "translation_key": self.translation_key, - "unique_id": self.unique_id, - "previous_unique_id": self.previous_unique_id, - "unit_of_measurement": self.unit_of_measurement, - } - ) + return cached_json_fragment( + { + "aliases": self.compat_aliases, + "aliases_v2": _serialize_aliases(self.aliases), + "area_id": self.area_id, + "categories": self.categories, + "capabilities": self.capabilities, + "config_entry_id": self.config_entry_id, + "config_subentry_id": self.config_subentry_id, + "created_at": self.created_at, + "device_class": self.device_class, + "device_id": self.device_id, + "disabled_by": self.disabled_by, + "entity_category": self.entity_category, + "entity_id": self.entity_id, + "hidden_by": self.hidden_by, + "icon": self.icon, + "id": self.id, + "has_entity_name": self.has_entity_name, + "labels": list(self.labels), + "modified_at": self.modified_at, + "name": self.name, + "object_id_base": self.object_id_base, + "options": self.options, + "original_device_class": self.original_device_class, + "original_icon": self.original_icon, + "original_name": self.original_name, + "platform": self.platform, + "suggested_object_id": self.suggested_object_id, + "supported_features": self.supported_features, + "translation_key": self.translation_key, + "unique_id": self.unique_id, + "previous_unique_id": self.previous_unique_id, + "unit_of_measurement": self.unit_of_measurement, + } ) @callback @@ -738,38 +744,36 @@ class DeletedRegistryEntry: @under_cached_property def as_storage_fragment(self) -> json_fragment: """Return a json fragment for storage.""" - return json_fragment( - json_bytes( - { - "aliases": self.compat_aliases, - "aliases_v2": _serialize_aliases(self.aliases), - "area_id": self.area_id, - "categories": self.categories, - "config_entry_id": self.config_entry_id, - "config_subentry_id": self.config_subentry_id, - "created_at": self.created_at, - "device_class": self.device_class, - "disabled_by": self.disabled_by - if self.disabled_by is not UNDEFINED - else None, - "disabled_by_undefined": self.disabled_by is UNDEFINED, - "entity_id": self.entity_id, - "hidden_by": self.hidden_by - if self.hidden_by is not UNDEFINED - else None, - "hidden_by_undefined": self.hidden_by is UNDEFINED, - "icon": self.icon, - "id": self.id, - "labels": list(self.labels), - "modified_at": self.modified_at, - "name": self.name, - "options": self.options if self.options is not UNDEFINED else {}, - "options_undefined": self.options is UNDEFINED, - "orphaned_timestamp": self.orphaned_timestamp, - "platform": self.platform, - "unique_id": self.unique_id, - } - ) + return cached_json_fragment( + { + "aliases": self.compat_aliases, + "aliases_v2": _serialize_aliases(self.aliases), + "area_id": self.area_id, + "categories": self.categories, + "config_entry_id": self.config_entry_id, + "config_subentry_id": self.config_subentry_id, + "created_at": self.created_at, + "device_class": self.device_class, + "disabled_by": self.disabled_by + if self.disabled_by is not UNDEFINED + else None, + "disabled_by_undefined": self.disabled_by is UNDEFINED, + "entity_id": self.entity_id, + "hidden_by": self.hidden_by + if self.hidden_by is not UNDEFINED + else None, + "hidden_by_undefined": self.hidden_by is UNDEFINED, + "icon": self.icon, + "id": self.id, + "labels": list(self.labels), + "modified_at": self.modified_at, + "name": self.name, + "options": self.options if self.options is not UNDEFINED else {}, + "options_undefined": self.options is UNDEFINED, + "orphaned_timestamp": self.orphaned_timestamp, + "platform": self.platform, + "unique_id": self.unique_id, + } ) diff --git a/homeassistant/helpers/json.py b/homeassistant/helpers/json.py index 3ea52348a095..0589490d2748 100644 --- a/homeassistant/helpers/json.py +++ b/homeassistant/helpers/json.py @@ -116,6 +116,28 @@ def json_bytes_strip_null(data: Any) -> bytes: json_fragment = orjson.Fragment +def cached_json_bytes(data: Any) -> bytes: + """Return json bytes right-sized for long-term caching. + + orjson over-allocates the returned bytes buffer and does not shrink it: the + logical length is set but the capacity is rounded up to a power of two (at + least a few KiB), so bytes cached for the lifetime of a long-lived object + retain several KiB of unused buffer. + """ + # Drop orjson's over-allocated slack with help of a memoryview. + return bytes(memoryview(json_bytes(data))) + + +def cached_json_fragment(data: Any) -> orjson.Fragment: + """Return a json fragment right-sized for long-term caching. + + Wraps the same right-sized bytes as cached_json_bytes; the body is inlined + rather than calling it to avoid an extra function call on this hot path. + """ + # Drop orjson's over-allocated slack with help of a memoryview. + return orjson.Fragment(bytes(memoryview(json_bytes(data)))) + + def json_dumps(data: Any) -> str: r"""Dump json string. diff --git a/tests/helpers/test_json.py b/tests/helpers/test_json.py index 9807e2a15526..ba43f0341706 100644 --- a/tests/helpers/test_json.py +++ b/tests/helpers/test_json.py @@ -1,12 +1,15 @@ """Test Home Assistant remote methods and classes.""" +from collections.abc import Callable import datetime from functools import partial +import gc import json import math import os from pathlib import Path import time +import tracemalloc from typing import Any, NamedTuple from unittest.mock import Mock, patch @@ -16,7 +19,10 @@ from homeassistant.core import Event, HomeAssistant, State from homeassistant.helpers.json import ( ExtendedJSONEncoder, JSONEncoder as DefaultHASSJSONEncoder, + cached_json_bytes, + cached_json_fragment, find_paths_unserializable_data, + json_bytes, json_bytes_sorted, json_bytes_strip_null, json_dumps, @@ -208,6 +214,70 @@ def test_json_fragments() -> None: ) +def test_cached_json_fragment() -> None: + """Test cached_json_fragment serializes identically to a plain fragment.""" + data = {"a": 1, "b": [1, 2, 3], "c": {"nested": True}, "d": None} + + fragment = cached_json_fragment(data) + assert isinstance(fragment, json_fragment) + assert json_dumps([fragment]) == json_dumps([json_fragment(json_bytes(data))]) + assert ( + json_dumps([fragment]) == '[{"a":1,"b":[1,2,3],"c":{"nested":true},"d":null}]' + ) + + +def test_cached_json_bytes() -> None: + """Test cached_json_bytes serializes identically to json_bytes.""" + data = {"a": 1, "b": [1, 2, 3], "c": {"nested": True}, "d": None} + + assert cached_json_bytes(data) == json_bytes(data) + assert ( + cached_json_bytes(data) == b'{"a":1,"b":[1,2,3],"c":{"nested":true},"d":null}' + ) + + +@pytest.mark.parametrize( + "cached_serializer", + [cached_json_bytes, cached_json_fragment], + ids=["cached_json_bytes", "cached_json_fragment"], +) +def test_cached_json_helpers_trim_buffer( + cached_serializer: Callable[[Any], object], +) -> None: + """Test the cached_json_* helpers cache right-sized bytes, not orjson's slack. + + orjson.dumps returns bytes whose backing buffer is rounded up to a power of + two and not shrunk; the helpers copy them to a right-sized buffer. Without + that copy the cached value would retain the full over-allocated buffer + (several KiB even for a small payload), which is the memory regression this + guards against. + + The waste is invisible to normal object inspection: sys.getsizeof() reports + the logical length, not the backing buffer, and orjson.Fragment exposes no way + to reach the bytes it wraps, so the retained allocation can only be observed + via tracemalloc. + """ + data = {f"key_{index}": "value" * 5 for index in range(40)} + serialized_size = len(json_bytes(data)) + + tracemalloc.start() + try: + # clear_traces resets the baseline to zero so pre-existing garbage from + # the test session is not counted; the transient over-allocated buffer is + # freed by refcounting before get_traced_memory, leaving only `cached`. + gc.collect() + tracemalloc.clear_traces() + cached = cached_serializer(data) + retained, _ = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert cached is not None # keep alive until measured + # The cache holds ~the serialized size; without the copy it would hold + # orjson's oversized power-of-two buffer, which is far larger. + assert retained < serialized_size * 1.5 + + def test_json_bytes_strip_null() -> None: """Test stripping nul from strings."""