TrueNAS/scale-build
1
0
Fork 0
mirror of https://github.com/truenas/scale-build.git synced 2025-12-20 02:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "NAS-132446 / 25.04 / Initial add of option to use internal APT repo for builds (#754)" (#759)

Browse Source
This commit is contained in:
Caleb St. John
2024-11-15 15:33:40 -05:00
committed by GitHub
parent a7fe3aa528
commit d8b7cacbf1
7 changed files with 20 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
conf/build.manifest
View File

@@ -8,25 +8,23 @@ identity_file_path_default: "~/.ssh/id_rsa"
# into the build chroots, or the final system images.
############################################################################
apt-repos:
base-url: https://apt.sys.truenas.net
base-url-internal: http://apt-mirror.tn.ixsystems.net
url: /fangtooth/nightlies/debian/
url: https://apt.sys.truenas.net/fangtooth/nightlies/debian/
distribution: bookworm
components: main
additional:
- url: /fangtooth/nightlies/debian-security/
- url: https://apt.sys.truenas.net/fangtooth/nightlies/debian-security/
distribution: bookworm-security
component: main
- url: /fangtooth/nightlies/debian-backports/
- url: https://apt.sys.truenas.net/fangtooth/nightlies/debian-backports/
distribution: bookworm-backports
component: "main contrib non-free non-free-firmware"
- url: /fangtooth/nightlies/debian-debug/
- url: https://apt.sys.truenas.net/fangtooth/nightlies/debian-debug/
distribution: bookworm-debug
component: main
- url: /fangtooth/nightlies/yarn/
- url: https://apt.sys.truenas.net/fangtooth/nightlies/yarn/
distribution: stable
component: main
- url: /fangtooth/nightlies/docker/
- url: https://apt.sys.truenas.net/fangtooth/nightlies/docker/
distribution: bookworm
component: stable
key: keys/docker.gpg

13
scale_build/bootstrap/bootstrapdir.py
View File

@@ -3,7 +3,7 @@ import os
import shutil
from scale_build.clean import clean_packages
from scale_build.utils.manifest import get_apt_base_url, get_manifest
from scale_build.utils.manifest import get_manifest
from scale_build.utils.paths import BUILDER_DIR, CHROOT_BASEDIR, REFERENCE_FILES, REFERENCE_FILES_DIR
from scale_build.utils.run import run
@@ -30,12 +30,11 @@ class BootstrapDir(CacheMixin, HashMixin):
def debootstrap_debian(self):
manifest = get_manifest()
apt_base_url = get_apt_base_url()
run(
['debootstrap'] + self.deopts + [
'--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg',
manifest['debian_release'],
self.chroot_basedir, apt_base_url + manifest['apt-repos']['url']
self.chroot_basedir, manifest['apt-repos']['url']
]
)
@@ -47,7 +46,6 @@ class BootstrapDir(CacheMixin, HashMixin):
self.add_trusted_apt_key()
apt_repos = get_manifest()['apt-repos']
apt_base_url = get_apt_base_url()
self.debootstrap_debian()
self.setup_mounts()
@@ -63,7 +61,7 @@ class BootstrapDir(CacheMixin, HashMixin):
run(['chroot', self.chroot_basedir, 'apt', 'install', '-y', 'gnupg'])
# Save the correct repo in sources.list
apt_sources = [f'deb {apt_base_url}{apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}']
apt_sources = [f'deb {apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}']
# Add additional repos
for repo in apt_repos['additional']:
@@ -73,7 +71,7 @@ class BootstrapDir(CacheMixin, HashMixin):
run(['chroot', self.chroot_basedir, 'apt-key', 'add', '/apt.key'])
os.unlink(os.path.join(self.chroot_basedir, 'apt.key'))
apt_sources.append(f'deb {apt_base_url}{repo["url"]} {repo["distribution"]} {repo["component"]}')
apt_sources.append(f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}')
with open(apt_sources_path, 'w') as f:
f.write('\n'.join(apt_sources))
@@ -149,12 +147,11 @@ class RootfsBootstrapDir(BootstrapDir):
def debootstrap_debian(self):
manifest = get_manifest()
apt_base_url = get_apt_base_url()
run(
['debootstrap'] + self.deopts + [
'--foreign', '--keyring', '/etc/apt/trusted.gpg.d/debian-archive-truenas-automatic.gpg',
manifest['debian_release'],
self.chroot_basedir, apt_base_url + manifest['apt-repos']['url']
self.chroot_basedir, manifest['apt-repos']['url']
]
)
for reference_file in REFERENCE_FILES:

7
scale_build/bootstrap/hash.py
View File

@@ -6,7 +6,7 @@ import re
import requests
import urllib.parse
from scale_build.utils.manifest import get_apt_base_url, get_manifest
from scale_build.utils.manifest import get_manifest
from scale_build.utils.run import run
from scale_build.utils.paths import CACHE_DIR, HASH_DIR
@@ -27,12 +27,11 @@ def get_repo_hash(repo_url, distribution):
def get_all_repo_hash():
apt_repos = get_manifest()['apt-repos']
apt_base_url = get_apt_base_url()
# Start by validating the main APT repo
all_repo_hash = get_repo_hash(apt_base_url + apt_repos['url'], apt_repos['distribution'])
all_repo_hash = get_repo_hash(apt_repos['url'], apt_repos['distribution'])
for repo_config in apt_repos['additional']:
all_repo_hash += get_repo_hash(apt_base_url + repo_config['url'], repo_config['distribution'])
all_repo_hash += get_repo_hash(repo_config['url'], repo_config['distribution'])
all_repo_hash += hashlib.sha256(get_apt_preferences().encode()).hexdigest()

2
scale_build/config.py
View File

@@ -49,8 +49,6 @@ TRY_BRANCH_OVERRIDE = get_env_variable('TRY_BRANCH_OVERRIDE', str)
VERSION = get_env_variable('TRUENAS_VERSION', str, f'{_VERS}-{BUILD_TIME_OBJ.strftime("%Y%m%d-%H%M%S")}')
TRUENAS_VENDOR = get_env_variable('TRUENAS_VENDOR', str)
PRESERVE_ISO = get_env_variable('PRESERVE_ISO', bool, False)
APT_INTERNAL_BUILD = get_env_variable('APT_INTERNAL_BUILD', bool, False)
APT_BASE_CUSTOM = get_env_variable('APT_BASE_CUSTOM', str)
# We will get branch overrides and identity file path overrides from here

6
scale_build/image/iso.py
View File

@@ -10,7 +10,7 @@ import json
import requests
from scale_build.exceptions import CallError
from scale_build.utils.manifest import get_apt_base_url, get_manifest
from scale_build.utils.manifest import get_manifest
from scale_build.utils.run import run
from scale_build.utils.paths import CD_DIR, CD_FILES_DIR, CHROOT_BASEDIR, CONF_GRUB, PKG_DIR, RELEASE_DIR, TMP_DIR
from scale_build.config import TRUENAS_VENDOR
@@ -134,10 +134,8 @@ def make_iso_file():
with tempfile.NamedTemporaryFile(dir=RELEASE_DIR) as efi_img:
with tempfile.NamedTemporaryFile(suffix='.tar.gz') as f:
apt_repos = get_manifest()['apt-repos']
apt_base_url = get_apt_base_url()
r = requests.get(
f'{apt_base_url}{apt_repos["url"]}dists/{apt_repos["distribution"]}'
'/main/installer-amd64/current/images/cdrom/'
f'{apt_repos["url"]}dists/{apt_repos["distribution"]}/main/installer-amd64/current/images/cdrom/'
'debian-cd_info.tar.gz',
timeout=10,
stream=True,

8
scale_build/image/update.py
View File

@@ -8,7 +8,7 @@ import stat
import tempfile
from scale_build.config import SIGNING_KEY, SIGNING_PASSWORD
from scale_build.utils.manifest import get_apt_base_url, get_manifest
from scale_build.utils.manifest import get_manifest
from scale_build.utils.run import run
from scale_build.utils.paths import CHROOT_BASEDIR, RELEASE_DIR, UPDATE_DIR
@@ -111,11 +111,9 @@ def install_rootfs_packages_impl():
def get_apt_sources():
apt_repos = get_manifest()['apt-repos']
apt_base_url = get_apt_base_url()
apt_sources = [f'deb {apt_base_url}{apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}']
apt_sources = [f'deb {apt_repos["url"]} {apt_repos["distribution"]} {apt_repos["components"]}']
for repo in apt_repos['additional']:
apt_sources.append(f'deb {apt_base_url}{repo["url"]} {repo["distribution"]} {repo["component"]}')
apt_sources.append(f'deb {repo["url"]} {repo["distribution"]} {repo["component"]}')
return apt_sources

20
scale_build/utils/manifest.py
View File

@@ -5,7 +5,7 @@ import yaml
from urllib.parse import urlparse
from scale_build.config import APT_BASE_CUSTOM, APT_INTERNAL_BUILD, SKIP_SOURCE_REPO_VALIDATION, TRAIN
from scale_build.config import SKIP_SOURCE_REPO_VALIDATION, TRAIN
from scale_build.exceptions import CallError, MissingManifest
from scale_build.utils.paths import MANIFEST
@@ -77,8 +77,6 @@ MANIFEST_SCHEMA = {
'code_name': {'type': 'string'},
'debian_release': {'type': 'string'},
'identity_file_path_default': {'type': 'string'},
'base-url': {'type': 'string'},
'base-url-internal': {'type': 'string'},
'apt-repos': {
'type': 'object',
'properties': {
@@ -254,19 +252,3 @@ def validate_manifest():
'accepts packages from github.com/truenas organization (To skip this for dev '
'purposes, please set "SKIP_SOURCE_REPO_VALIDATION" in your environment).'
)
def get_apt_base_url():
apt_repos = get_manifest()['apt-repos']
apt_base_url = ""
# If the user provided their own location
if APT_BASE_CUSTOM:
return APT_BASE_CUSTOM
# Return either the CDN or the internal build url
if APT_INTERNAL_BUILD:
apt_base_url = f'{apt_repos["base-url-internal"]}'
else:
apt_base_url = f'{apt_repos["base-url"]}'
return apt_base_url