Fall back to PyPI when the extra index is down (#175404)

homeassistant/util/package.py

@@ -123,6 +123,23 @@ _UV_ENV_PYTHON_VARS = (
 )
 
 
+def _install(args: list[str], env: dict[str, str]) -> str | None:
+    """Run the install command and return stderr output if it failed."""
+    _LOGGER.debug("Running uv pip command: args=%s", args)
+    with Popen(
+        args,
+        stdin=PIPE,
+        stdout=PIPE,
+        stderr=PIPE,
+        env=env,
+        close_fds=False,  # required for posix_spawn
+    ) as process:
+        _, stderr = process.communicate()
+        if process.returncode != 0:
+            return stderr.decode("utf-8").lstrip().strip()
+    return None
+
+
 def install_package(
     package: str,
     upgrade: bool = True,
@@ -171,25 +188,39 @@ def install_package(
         # https://github.com/astral-sh/uv/issues/2077#issuecomment-2150406001
         args += ["--python", sys.executable, "--target", abs_target]
 
-    _LOGGER.debug("Running uv pip command: args=%s", args)
-    with Popen(
-        args,
-        stdin=PIPE,
-        stdout=PIPE,
-        stderr=PIPE,
-        env=env,
-        close_fds=False,  # required for posix_spawn
-    ) as process:
-        _, stderr = process.communicate()
-        if process.returncode != 0:
-            _LOGGER.error(
-                "Unable to install package %s: %s",
-                package,
-                stderr.decode("utf-8").lstrip().strip(),
-            )
-            return False
+    if (stderr := _install(args, env)) is None:
+        return True
 
-    return True
+    # uv treats a failing extra index as fatal, unlike pip which skips it.
+    # If the error came from an extra index host (e.g. the wheels index
+    # being down), retry with only the healthy indexes so PyPI can serve
+    # the package. Match on the host since wheel files may live outside
+    # the index path.
+    extra_urls = env.get("UV_EXTRA_INDEX_URL", "").split()
+    # Log only the hosts since the URLs may contain credentials
+    failing = {
+        url: host
+        for url in extra_urls
+        if (host := urlparse(url).hostname) and host in stderr
+    }
+    if failing:
+        _LOGGER.warning(
+            "Unable to install package %s using extra index host %s: %s; "
+            "retrying without it",
+            package,
+            ", ".join(failing.values()),
+            stderr,
+        )
+        retry_env = env.copy()
+        if remaining := [url for url in extra_urls if url not in failing]:
+            retry_env["UV_EXTRA_INDEX_URL"] = " ".join(remaining)
+        else:
+            del retry_env["UV_EXTRA_INDEX_URL"]
+        if (stderr := _install(args, retry_env)) is None:
+            return True
+
+    _LOGGER.error("Unable to install package %s: %s", package, stderr)
+    return False
 
 
 async def async_get_user_site(deps_dir: str) -> str:

tests/util/test_package.py

@@ -7,7 +7,7 @@ import logging
 import os
 from subprocess import PIPE
 import sys
-from unittest.mock import MagicMock, Mock, call, patch
+from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
 
 import pytest
 
@@ -21,6 +21,16 @@ TEST_NEW_REQ = "pyhelloworld3==1.0.0"
 
 TEST_ZIP_REQ = f"file://{RESOURCE_DIR}/pyhelloworld3.zip#{TEST_NEW_REQ}"
 
+TEST_EXTRA_INDEX_URL = "https://wheels.home-assistant.io/musllinux-index/"
+TEST_OTHER_INDEX_URL = "https://example.com/simple/"
+TEST_INDEX_FAIL_STDERR = (
+    f"error: Failed to fetch: `{TEST_EXTRA_INDEX_URL}pyhelloworld3/`"
+)
+TEST_WHEEL_FAIL_STDERR = (
+    "error: Failed to fetch: `https://wheels.home-assistant.io/"
+    "pyhelloworld3-1.0.0-cp314-cp314-musllinux_1_2_x86_64.whl`"
+)
+
 
 @pytest.fixture
 def mock_sys() -> Generator[MagicMock]:
@@ -308,6 +318,103 @@ def test_install_error(caplog: pytest.LogCaptureFixture, mock_popen) -> None:
         assert record.levelname == "ERROR"
 
 
+@pytest.mark.parametrize(
+    ("extra_index", "install_stderr", "expected_retry_env"),
+    [
+        pytest.param(
+            TEST_EXTRA_INDEX_URL, TEST_INDEX_FAIL_STDERR, {}, id="single_index_removed"
+        ),
+        pytest.param(
+            f"{TEST_EXTRA_INDEX_URL} {TEST_OTHER_INDEX_URL}",
+            TEST_INDEX_FAIL_STDERR,
+            {"UV_EXTRA_INDEX_URL": TEST_OTHER_INDEX_URL},
+            id="healthy_index_kept",
+        ),
+        pytest.param(
+            TEST_EXTRA_INDEX_URL,
+            TEST_WHEEL_FAIL_STDERR,
+            {},
+            id="wheel_file_outside_index_path",
+        ),
+    ],
+)
+@pytest.mark.usefixtures("mock_sys", "mock_venv")
+def test_install_extra_index_fallback(
+    mock_popen: MagicMock,
+    mock_env_copy: Mock,
+    extra_index: str,
+    install_stderr: str,
+    expected_retry_env: dict[str, str],
+) -> None:
+    """Test install retries without a failing extra index."""
+    env = mock_env_copy()
+    env["UV_EXTRA_INDEX_URL"] = extra_index
+    mock_popen.return_value.communicate.side_effect = [
+        (b"", install_stderr.encode()),
+        (b"", b""),
+    ]
+    type(mock_popen.return_value).returncode = PropertyMock(side_effect=[1, 0])
+    assert package.install_package(TEST_NEW_REQ, False)
+    assert mock_popen.return_value.communicate.call_count == 2
+    assert mock_popen.call_args_list[2].kwargs["env"] == expected_retry_env
+
+
+@pytest.mark.usefixtures("mock_sys", "mock_venv")
+def test_install_extra_index_fallback_redacts_credentials(
+    mock_popen: MagicMock,
+    mock_env_copy: Mock,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test the retry warning does not log extra index credentials."""
+    caplog.set_level(logging.WARNING)
+    env = mock_env_copy()
+    env["UV_EXTRA_INDEX_URL"] = (
+        "https://user:secret@wheels.home-assistant.io/musllinux-index/"
+    )
+    mock_popen.return_value.communicate.side_effect = [
+        (b"", TEST_INDEX_FAIL_STDERR.encode()),
+        (b"", b""),
+    ]
+    type(mock_popen.return_value).returncode = PropertyMock(side_effect=[1, 0])
+    assert package.install_package(TEST_NEW_REQ, False)
+    assert "wheels.home-assistant.io" in caplog.text
+    assert "secret" not in caplog.text
+
+
+@pytest.mark.usefixtures("mock_sys", "mock_venv")
+def test_install_extra_index_fallback_fails(
+    mock_popen: MagicMock,
+    mock_env_copy: Mock,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test install fails when the retry without the extra index also fails."""
+    caplog.set_level(logging.WARNING)
+    env = mock_env_copy()
+    env["UV_EXTRA_INDEX_URL"] = TEST_EXTRA_INDEX_URL
+    mock_popen.return_value.communicate.return_value = (
+        b"",
+        TEST_INDEX_FAIL_STDERR.encode(),
+    )
+    mock_popen.return_value.returncode = 1
+    assert not package.install_package(TEST_NEW_REQ, False)
+    assert mock_popen.return_value.communicate.call_count == 2
+    assert "retrying without it" in caplog.text
+    assert "Unable to install package" in caplog.text
+
+
+@pytest.mark.usefixtures("mock_sys", "mock_venv")
+def test_install_extra_index_unrelated_error(
+    mock_popen: MagicMock,
+    mock_env_copy: Mock,
+) -> None:
+    """Test install does not retry when the error is not from the extra index."""
+    env = mock_env_copy()
+    env["UV_EXTRA_INDEX_URL"] = TEST_EXTRA_INDEX_URL
+    mock_popen.return_value.returncode = 1
+    assert not package.install_package(TEST_NEW_REQ, False)
+    assert mock_popen.return_value.communicate.call_count == 1
+
+
 @pytest.mark.usefixtures("mock_venv")
 def test_install_constraint(mock_popen, mock_env_copy, mock_sys) -> None:
     """Test install with constraint file on not installed package."""