Home-Assistant/core
1
0
Fork 0
mirror of https://github.com/home-assistant/core.git synced 2026-09-13 20:20:10 +01:00
Code Activity

Trim cached orjson fragments kept by registries (#181240)

Browse Source 
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Erik Montnemery
2026-09-07 09:38:44 +02:00
committed by GitHub
co-authored by Copilot Autofix powered by AI
parent 88d5dab64d
commit 9afabaaba1
5 changed files with 254 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
homeassistant/helpers/area_registry.py
+15 -17
View File
@@ -14,7 +14,7 @@ from homeassistant.util.event_type import EventType
from homeassistant.util.hass_dict import HassKey
from . import device_registry as dr
from .json import json_bytes, json_fragment
from .json import cached_json_fragment, json_fragment
from .normalized_name_base_registry import (
NormalizedNameBaseRegistryEntry,
NormalizedNameBaseRegistryItems,
@@ -87,22 +87,20 @@ class AreaEntry(NormalizedNameBaseRegistryEntry):
@under_cached_property
def json_fragment(self) -> json_fragment:
"""Return a JSON representation of this AreaEntry."""
return json_fragment(
json_bytes(
{
"aliases": list(self.aliases),
"area_id": self.id,
"floor_id": self.floor_id,
"humidity_entity_id": self.humidity_entity_id,
"icon": self.icon,
"labels": list(self.labels),
"name": self.name,
"picture": self.picture,
"temperature_entity_id": self.temperature_entity_id,
"created_at": self.created_at.timestamp(),
"modified_at": self.modified_at.timestamp(),
}
)
return cached_json_fragment(
{
"aliases": list(self.aliases),
"area_id": self.id,
"floor_id": self.floor_id,
"humidity_entity_id": self.humidity_entity_id,
"icon": self.icon,
"labels": list(self.labels),
"name": self.name,
"picture": self.picture,
"temperature_entity_id": self.temperature_entity_id,
"created_at": self.created_at.timestamp(),
"modified_at": self.modified_at.timestamp(),
}
)
homeassistant/helpers/device_registry.py
+71 -73
View File
@@ -52,7 +52,13 @@ from .frame import (
get_integration_frame,
report_usage,
)
from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes, json_fragment
from .json import (
JSON_DUMP,
cached_json_bytes,
cached_json_fragment,
find_paths_unserializable_data,
json_fragment,
)
from .registry import BaseRegistry, BaseRegistryItems, RegistryIndexType
from .typing import UNDEFINED, UndefinedType
@@ -435,7 +441,7 @@ class BaseDeviceEntry:
"""Return a cached JSON representation of the entry."""
try:
dict_repr = self.dict_repr
return json_bytes(dict_repr)
return cached_json_bytes(dict_repr)
except ValueError, TypeError:
_LOGGER.error(
"Unable to serialize entry %s to JSON. Bad data found at %s",
@@ -564,39 +570,35 @@ class DeviceEntry(BaseDeviceEntry):
@under_cached_property
def as_storage_fragment(self) -> json_fragment:
"""Return a json fragment for storage."""
return json_fragment(
json_bytes(
{
"area_id": self.area_id,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"configuration_url": self.configuration_url,
"connections": list(self.connections),
"created_at": self.created_at,
"disabled_by": self.disabled_by,
"entry_type": self.entry_type,
"hw_version": self.hw_version,
"id": self.id,
"identifiers": list(self.identifiers),
"labels": list(self.labels),
"composite_device_id": self.composite_device_id,
"composite_primary_config_entry": (
self.composite_primary_config_entry
),
"split_at": self.split_at,
"manufacturer": self.manufacturer,
"model": self.model,
"model_id": self.model_id,
"modified_at": self.modified_at,
"name_by_user": self.name_by_user,
"name": self.name,
"has_composite_identifiers": (self.has_composite_identifiers),
"primary_config_entry": self.primary_config_entry,
"serial_number": self.serial_number,
"sw_version": self.sw_version,
"via_device_id": self.via_device_id,
}
)
return cached_json_fragment(
{
"area_id": self.area_id,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"configuration_url": self.configuration_url,
"connections": list(self.connections),
"created_at": self.created_at,
"disabled_by": self.disabled_by,
"entry_type": self.entry_type,
"hw_version": self.hw_version,
"id": self.id,
"identifiers": list(self.identifiers),
"labels": list(self.labels),
"composite_device_id": self.composite_device_id,
"composite_primary_config_entry": self.composite_primary_config_entry,
"split_at": self.split_at,
"manufacturer": self.manufacturer,
"model": self.model,
"model_id": self.model_id,
"modified_at": self.modified_at,
"name_by_user": self.name_by_user,
"name": self.name,
"has_composite_identifiers": (self.has_composite_identifiers),
"primary_config_entry": self.primary_config_entry,
"serial_number": self.serial_number,
"sw_version": self.sw_version,
"via_device_id": self.via_device_id,
}
)
@property
@@ -686,23 +688,21 @@ class ChildDeviceEntry(BaseDeviceEntry):
@under_cached_property
def as_storage_fragment(self) -> json_fragment:
"""Return a json fragment for storage."""
return json_fragment(
json_bytes(
{
"area_id": self.area_id,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"created_at": self.created_at,
"disabled_by": self.disabled_by,
"id": self.id,
"identifiers": list(self.identifiers),
"labels": list(self.labels),
"modified_at": self.modified_at,
"name_by_user": self.name_by_user,
"name": self.name,
"parent_device_id": self.parent_device_id,
}
)
return cached_json_fragment(
{
"area_id": self.area_id,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"created_at": self.created_at,
"disabled_by": self.disabled_by,
"id": self.id,
"identifiers": list(self.identifiers),
"labels": list(self.labels),
"modified_at": self.modified_at,
"name_by_user": self.name_by_user,
"name": self.name,
"parent_device_id": self.parent_device_id,
}
)
@@ -849,27 +849,25 @@ class DeletedDeviceEntry:
@under_cached_property
def as_storage_fragment(self) -> json_fragment:
"""Return a json fragment for storage."""
return json_fragment(
json_bytes(
{
"area_id": self.area_id,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"connections": list(self.connections),
"created_at": self.created_at,
"disabled_by": self.disabled_by
if self.disabled_by is not UNDEFINED
else None,
"disabled_by_undefined": self.disabled_by is UNDEFINED,
"identifiers": list(self.identifiers),
"id": self.id,
"labels": list(self.labels),
"modified_at": self.modified_at,
"name_by_user": self.name_by_user,
"orphaned_timestamp": self.orphaned_timestamp,
"domain": self.domain,
}
)
return cached_json_fragment(
{
"area_id": self.area_id,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"connections": list(self.connections),
"created_at": self.created_at,
"disabled_by": self.disabled_by
if self.disabled_by is not UNDEFINED
else None,
"disabled_by_undefined": self.disabled_by is UNDEFINED,
"identifiers": list(self.identifiers),
"id": self.id,
"labels": list(self.labels),
"modified_at": self.modified_at,
"name_by_user": self.name_by_user,
"orphaned_timestamp": self.orphaned_timestamp,
"domain": self.domain,
}
)
homeassistant/helpers/entity_registry.py
+76 -72
View File
@@ -53,7 +53,13 @@ from .device_registry import (
EventDeviceRegistryUpdatedData,
)
from .frame import ReportBehavior, report_usage
from .json import JSON_DUMP, find_paths_unserializable_data, json_bytes, json_fragment
from .json import (
JSON_DUMP,
cached_json_bytes,
cached_json_fragment,
find_paths_unserializable_data,
json_fragment,
)
from .registry import BaseRegistry, BaseRegistryItems, RegistryIndexType
from .singleton import singleton
from .typing import UNDEFINED, UndefinedType
@@ -319,7 +325,9 @@ class RegistryEntry:
"""
try:
dict_repr = self._as_display_dict
json_repr: bytes | None = json_bytes(dict_repr) if dict_repr else None
json_repr: bytes | None = (
cached_json_bytes(dict_repr) if dict_repr else None
)
except ValueError, TypeError:
_LOGGER.error(
"Unable to serialize entry %s to JSON. Bad data found at %s",
@@ -386,7 +394,7 @@ class RegistryEntry:
"""Return a cached partial JSON representation of the entry."""
try:
dict_repr = self.as_partial_dict
return json_bytes(dict_repr)
return cached_json_bytes(dict_repr)
except ValueError, TypeError:
_LOGGER.error(
"Unable to serialize entry %s to JSON. Bad data found at %s",
@@ -400,43 +408,41 @@ class RegistryEntry:
@under_cached_property
def as_storage_fragment(self) -> json_fragment:
"""Return a json fragment for storage."""
return json_fragment(
json_bytes(
{
"aliases": self.compat_aliases,
"aliases_v2": _serialize_aliases(self.aliases),
"area_id": self.area_id,
"categories": self.categories,
"capabilities": self.capabilities,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"created_at": self.created_at,
"device_class": self.device_class,
"device_id": self.device_id,
"disabled_by": self.disabled_by,
"entity_category": self.entity_category,
"entity_id": self.entity_id,
"hidden_by": self.hidden_by,
"icon": self.icon,
"id": self.id,
"has_entity_name": self.has_entity_name,
"labels": list(self.labels),
"modified_at": self.modified_at,
"name": self.name,
"object_id_base": self.object_id_base,
"options": self.options,
"original_device_class": self.original_device_class,
"original_icon": self.original_icon,
"original_name": self.original_name,
"platform": self.platform,
"suggested_object_id": self.suggested_object_id,
"supported_features": self.supported_features,
"translation_key": self.translation_key,
"unique_id": self.unique_id,
"previous_unique_id": self.previous_unique_id,
"unit_of_measurement": self.unit_of_measurement,
}
)
return cached_json_fragment(
{
"aliases": self.compat_aliases,
"aliases_v2": _serialize_aliases(self.aliases),
"area_id": self.area_id,
"categories": self.categories,
"capabilities": self.capabilities,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"created_at": self.created_at,
"device_class": self.device_class,
"device_id": self.device_id,
"disabled_by": self.disabled_by,
"entity_category": self.entity_category,
"entity_id": self.entity_id,
"hidden_by": self.hidden_by,
"icon": self.icon,
"id": self.id,
"has_entity_name": self.has_entity_name,
"labels": list(self.labels),
"modified_at": self.modified_at,
"name": self.name,
"object_id_base": self.object_id_base,
"options": self.options,
"original_device_class": self.original_device_class,
"original_icon": self.original_icon,
"original_name": self.original_name,
"platform": self.platform,
"suggested_object_id": self.suggested_object_id,
"supported_features": self.supported_features,
"translation_key": self.translation_key,
"unique_id": self.unique_id,
"previous_unique_id": self.previous_unique_id,
"unit_of_measurement": self.unit_of_measurement,
}
)
@callback
@@ -738,38 +744,36 @@ class DeletedRegistryEntry:
@under_cached_property
def as_storage_fragment(self) -> json_fragment:
"""Return a json fragment for storage."""
return json_fragment(
json_bytes(
{
"aliases": self.compat_aliases,
"aliases_v2": _serialize_aliases(self.aliases),
"area_id": self.area_id,
"categories": self.categories,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"created_at": self.created_at,
"device_class": self.device_class,
"disabled_by": self.disabled_by
if self.disabled_by is not UNDEFINED
else None,
"disabled_by_undefined": self.disabled_by is UNDEFINED,
"entity_id": self.entity_id,
"hidden_by": self.hidden_by
if self.hidden_by is not UNDEFINED
else None,
"hidden_by_undefined": self.hidden_by is UNDEFINED,
"icon": self.icon,
"id": self.id,
"labels": list(self.labels),
"modified_at": self.modified_at,
"name": self.name,
"options": self.options if self.options is not UNDEFINED else {},
"options_undefined": self.options is UNDEFINED,
"orphaned_timestamp": self.orphaned_timestamp,
"platform": self.platform,
"unique_id": self.unique_id,
}
)
return cached_json_fragment(
{
"aliases": self.compat_aliases,
"aliases_v2": _serialize_aliases(self.aliases),
"area_id": self.area_id,
"categories": self.categories,
"config_entry_id": self.config_entry_id,
"config_subentry_id": self.config_subentry_id,
"created_at": self.created_at,
"device_class": self.device_class,
"disabled_by": self.disabled_by
if self.disabled_by is not UNDEFINED
else None,
"disabled_by_undefined": self.disabled_by is UNDEFINED,
"entity_id": self.entity_id,
"hidden_by": self.hidden_by
if self.hidden_by is not UNDEFINED
else None,
"hidden_by_undefined": self.hidden_by is UNDEFINED,
"icon": self.icon,
"id": self.id,
"labels": list(self.labels),
"modified_at": self.modified_at,
"name": self.name,
"options": self.options if self.options is not UNDEFINED else {},
"options_undefined": self.options is UNDEFINED,
"orphaned_timestamp": self.orphaned_timestamp,
"platform": self.platform,
"unique_id": self.unique_id,
}
)
homeassistant/helpers/json.py
+22
View File
@@ -116,6 +116,28 @@ def json_bytes_strip_null(data: Any) -> bytes:
json_fragment = orjson.Fragment
def cached_json_bytes(data: Any) -> bytes:
"""Return json bytes right-sized for long-term caching.
orjson over-allocates the returned bytes buffer and does not shrink it: the
logical length is set but the capacity is rounded up to a power of two (at
least a few KiB), so bytes cached for the lifetime of a long-lived object
retain several KiB of unused buffer.
"""
# Drop orjson's over-allocated slack with help of a memoryview.
return bytes(memoryview(json_bytes(data)))
def cached_json_fragment(data: Any) -> orjson.Fragment:
"""Return a json fragment right-sized for long-term caching.
Wraps the same right-sized bytes as cached_json_bytes; the body is inlined
rather than calling it to avoid an extra function call on this hot path.
"""
# Drop orjson's over-allocated slack with help of a memoryview.
return orjson.Fragment(bytes(memoryview(json_bytes(data))))
def json_dumps(data: Any) -> str:
r"""Dump json string.
tests/helpers/test_json.py
+70
View File
@@ -1,12 +1,15 @@
"""Test Home Assistant remote methods and classes."""
from collections.abc import Callable
import datetime
from functools import partial
import gc
import json
import math
import os
from pathlib import Path
import time
import tracemalloc
from typing import Any, NamedTuple
from unittest.mock import Mock, patch
@@ -16,7 +19,10 @@ from homeassistant.core import Event, HomeAssistant, State
from homeassistant.helpers.json import (
ExtendedJSONEncoder,
JSONEncoder as DefaultHASSJSONEncoder,
cached_json_bytes,
cached_json_fragment,
find_paths_unserializable_data,
json_bytes,
json_bytes_sorted,
json_bytes_strip_null,
json_dumps,
@@ -208,6 +214,70 @@ def test_json_fragments() -> None:
)
def test_cached_json_fragment() -> None:
"""Test cached_json_fragment serializes identically to a plain fragment."""
data = {"a": 1, "b": [1, 2, 3], "c": {"nested": True}, "d": None}
fragment = cached_json_fragment(data)
assert isinstance(fragment, json_fragment)
assert json_dumps([fragment]) == json_dumps([json_fragment(json_bytes(data))])
assert (
json_dumps([fragment]) == '[{"a":1,"b":[1,2,3],"c":{"nested":true},"d":null}]'
)
def test_cached_json_bytes() -> None:
"""Test cached_json_bytes serializes identically to json_bytes."""
data = {"a": 1, "b": [1, 2, 3], "c": {"nested": True}, "d": None}
assert cached_json_bytes(data) == json_bytes(data)
assert (
cached_json_bytes(data) == b'{"a":1,"b":[1,2,3],"c":{"nested":true},"d":null}'
)
@pytest.mark.parametrize(
"cached_serializer",
[cached_json_bytes, cached_json_fragment],
ids=["cached_json_bytes", "cached_json_fragment"],
)
def test_cached_json_helpers_trim_buffer(
cached_serializer: Callable[[Any], object],
) -> None:
"""Test the cached_json_* helpers cache right-sized bytes, not orjson's slack.
orjson.dumps returns bytes whose backing buffer is rounded up to a power of
two and not shrunk; the helpers copy them to a right-sized buffer. Without
that copy the cached value would retain the full over-allocated buffer
(several KiB even for a small payload), which is the memory regression this
guards against.
The waste is invisible to normal object inspection: sys.getsizeof() reports
the logical length, not the backing buffer, and orjson.Fragment exposes no way
to reach the bytes it wraps, so the retained allocation can only be observed
via tracemalloc.
"""
data = {f"key_{index}": "value" * 5 for index in range(40)}
serialized_size = len(json_bytes(data))
tracemalloc.start()
try:
# clear_traces resets the baseline to zero so pre-existing garbage from
# the test session is not counted; the transient over-allocated buffer is
# freed by refcounting before get_traced_memory, leaving only `cached`.
gc.collect()
tracemalloc.clear_traces()
cached = cached_serializer(data)
retained, _ = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert cached is not None # keep alive until measured
# The cache holds ~the serialized size; without the copy it would hold
# orjson's oversized power-of-two buffer, which is far larger.
assert retained < serialized_size * 1.5
def test_json_bytes_strip_null() -> None:
"""Test stripping nul from strings."""