Home-Assistant/core
1
0
Fork 0
mirror of https://github.com/home-assistant/core.git synced 2025-12-27 14:31:13 +00:00
Code Activity

Generate requirements per supported architecture (#115708)

Browse Source 
* Generate requirements per supported architecture

* Don't store wheels requirements in the repo

* Dry run

* Set Python version

* Install base packages

* Fix

* Fix

* Fix

* Fix typo

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

* Genarate requirements_all_pytest.txt

* Fix hassfest

* Reenable building wheels

* Remove unneeded code

* Address review comment

* Fix lying comment

* Add tests, address review comments

* Deduplicate

* Fix file name

* Add comment

---------

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>
This commit is contained in:
Erik Montnemery
2024-04-22 19:23:08 +02:00
committed by GitHub
parent 124eca4d53
commit 2caca7fbe3
6 changed files with 165 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
script/gen_requirements_all.py
View File

@@ -17,7 +17,10 @@ from typing import Any
from homeassistant.util.yaml.loader import load_yaml
from script.hassfest.model import Integration
COMMENT_REQUIREMENTS = (
# Requirements which can't be installed on all systems because they rely on additional
# system packages. Requirements listed in EXCLUDED_REQUIREMENTS_ALL will be commented-out
# in requirements_all.txt and requirements_test_all.txt.
EXCLUDED_REQUIREMENTS_ALL = {
"atenpdu", # depends on pysnmp which is not maintained at this time
"avea", # depends on bluepy
"avion",
@@ -36,10 +39,39 @@ COMMENT_REQUIREMENTS = (
"pyuserinput",
"tensorflow",
"tf-models-official",
)
}
COMMENT_REQUIREMENTS_NORMALIZED = {
commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
# Requirements excluded by EXCLUDED_REQUIREMENTS_ALL which should be included when
# building integration wheels for all architectures.
INCLUDED_REQUIREMENTS_WHEELS = {
"decora-wifi",
"evdev",
"pycups",
"python-gammu",
"pyuserinput",
}
# Requirements to exclude or include when running github actions.
# Requirements listed in "exclude" will be commented-out in
# requirements_all_{action}.txt
# Requirements listed in "include" must be listed in EXCLUDED_REQUIREMENTS_CI, and
# will be included in requirements_all_{action}.txt
OVERRIDDEN_REQUIREMENTS_ACTIONS = {
"pytest": {"exclude": set(), "include": {"python-gammu"}},
"wheels_aarch64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
# Pandas has issues building on armhf, it is expected they
# will drop the platform in the near future (they consider it
# "flimsy" on 386). The following packages depend on pandas,
# so we comment them out.
"wheels_armhf": {
"exclude": {"env-canada", "noaa-coops", "pyezviz", "pykrakenapi"},
"include": INCLUDED_REQUIREMENTS_WHEELS,
},
"wheels_armv7": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
"wheels_amd64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
"wheels_i386": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
}
IGNORE_PIN = ("colorlog>2.1,<3", "urllib3")
@@ -254,6 +286,12 @@ def gather_recursive_requirements(
return reqs
def _normalize_package_name(package_name: str) -> str:
"""Normalize a package name."""
# pipdeptree needs lowercase and dash instead of underscore or period as separator
return package_name.lower().replace("_", "-").replace(".", "-")
def normalize_package_name(requirement: str) -> str:
"""Return a normalized package name from a requirement string."""
# This function is also used in hassfest.
@@ -262,12 +300,24 @@ def normalize_package_name(requirement: str) -> str:
return ""
# pipdeptree needs lowercase and dash instead of underscore or period as separator
return match.group(1).lower().replace("_", "-").replace(".", "-")
return _normalize_package_name(match.group(1))
def comment_requirement(req: str) -> bool:
"""Comment out requirement. Some don't install on all systems."""
return normalize_package_name(req) in COMMENT_REQUIREMENTS_NORMALIZED
return normalize_package_name(req) in EXCLUDED_REQUIREMENTS_ALL
def process_action_requirement(req: str, action: str) -> str:
"""Process requirement for a specific github action."""
normalized_package_name = normalize_package_name(req)
if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["exclude"]:
return f"# {req}"
if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["include"]:
return req
if normalized_package_name in EXCLUDED_REQUIREMENTS_ALL:
return f"# {req}"
return req
def gather_modules() -> dict[str, list[str]] | None:
@@ -353,6 +403,16 @@ def generate_requirements_list(reqs: dict[str, list[str]]) -> str:
return "".join(output)
def generate_action_requirements_list(reqs: dict[str, list[str]], action: str) -> str:
"""Generate a pip file based on requirements."""
output = []
for pkg, requirements in sorted(reqs.items(), key=itemgetter(0)):
output.extend(f"\n# {req}" for req in sorted(requirements))
processed_pkg = process_action_requirement(pkg, action)
output.append(f"\n{processed_pkg}\n")
return "".join(output)
def requirements_output() -> str:
"""Generate output for requirements."""
output = [
@@ -379,6 +439,18 @@ def requirements_all_output(reqs: dict[str, list[str]]) -> str:
return "".join(output)
def requirements_all_action_output(reqs: dict[str, list[str]], action: str) -> str:
"""Generate output for requirements_all_{action}."""
output = [
f"# Home Assistant Core, full dependency set for {action}\n",
GENERATED_MESSAGE,
"-r requirements.txt\n",
]
output.append(generate_action_requirements_list(reqs, action))
return "".join(output)
def requirements_test_all_output(reqs: dict[str, list[str]]) -> str:
"""Generate output for test_requirements."""
output = [
@@ -459,7 +531,7 @@ def diff_file(filename: str, content: str) -> list[str]:
)
def main(validate: bool) -> int:
def main(validate: bool, ci: bool) -> int:
"""Run the script."""
if not os.path.isfile("requirements_all.txt"):
print("Run this from HA root dir")
@@ -472,17 +544,28 @@ def main(validate: bool) -> int:
reqs_file = requirements_output()
reqs_all_file = requirements_all_output(data)
reqs_all_action_files = {
action: requirements_all_action_output(data, action)
for action in OVERRIDDEN_REQUIREMENTS_ACTIONS
}
reqs_test_all_file = requirements_test_all_output(data)
# Always calling requirements_pre_commit_output is intentional to ensure
# the code is called by the pre-commit hooks.
reqs_pre_commit_file = requirements_pre_commit_output()
constraints = gather_constraints()
files = (
files = [
("requirements.txt", reqs_file),
("requirements_all.txt", reqs_all_file),
("requirements_test_pre_commit.txt", reqs_pre_commit_file),
("requirements_test_all.txt", reqs_test_all_file),
("homeassistant/package_constraints.txt", constraints),
)
]
if ci:
files.extend(
(f"requirements_all_{action}.txt", reqs_all_file)
for action, reqs_all_file in reqs_all_action_files.items()
)
if validate:
errors = []
@@ -511,4 +594,5 @@ def main(validate: bool) -> int:
if __name__ == "__main__":
_VAL = sys.argv[-1] == "validate"
sys.exit(main(_VAL))
_CI = sys.argv[-1] == "ci"
sys.exit(main(_VAL, _CI))

10
script/hassfest/requirements.py
View File

@@ -15,13 +15,13 @@ from awesomeversion import AwesomeVersion, AwesomeVersionStrategy
from tqdm import tqdm
import homeassistant.util.package as pkg_util
from script.gen_requirements_all import COMMENT_REQUIREMENTS, normalize_package_name
from script.gen_requirements_all import (
EXCLUDED_REQUIREMENTS_ALL,
normalize_package_name,
)
from .model import Config, Integration
IGNORE_PACKAGES = {
commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
}
PACKAGE_REGEX = re.compile(
r"^(?:--.+\s)?([-_,\.\w\d\[\]]+)(==|>=|<=|~=|!=|<|>|===)*(.*)$"
)
@@ -116,7 +116,7 @@ def validate_requirements(integration: Integration) -> None:
f"Failed to normalize package name from requirement {req}",
)
return
if package in IGNORE_PACKAGES:
if package in EXCLUDED_REQUIREMENTS_ALL:
continue
integration_requirements.add(req)
integration_packages.add(package)