NAS-124055 / 24.04 / Implement dataset-based filesystem hierarchy (#494)

Create filesystem hierarchy for new boot environments based
on specifications in fhs.py file in truenas_install directory.
Exact spec for datasets is something we will fine-time as time
goes on.

This gives us more flexibility regarding which parts of FS to
make readonly, and also identifying precise local changes from
a pristine environment that users have made to system files.

Precise settings detailed in comments in the specification file.
Overall, this gives better posture for STIG compliance regarding
auditability and prevention of unauthorized OS changes.

scale_build/image/update.py

@@ -160,6 +160,13 @@ def custom_rootfs_setup():
     shutil.rmtree(tmp_systemd)
     run_in_chroot(['depmod'], check=False)
 
+    # /usr will be readonly and so we want the ca-certificates directory to
+    # symlink to writeable location in /var/local
+    local_cacerts = os.path.join(CHROOT_BASEDIR, "usr/local/share/ca-certificates")
+    os.makedirs(os.path.join(CHROOT_BASEDIR, "usr/local/share"), exist_ok=True)
+    shutil.rmtree(local_cacerts, ignore_errors=True)
+    os.symlink("/var/local/ca-certificates", local_cacerts)
+
 
 def clean_rootfs():
     to_remove = get_manifest()['base-prune']

scale_build/main.py

@@ -69,7 +69,7 @@ def main():
     )
 
     validate_parser = subparsers.add_parser('validate', help='Validate TrueNAS Scale build manifest and system state')
-    for action in ('manifest', 'system_state'):
+    for action in ('datasets', 'manifest', 'system_state'):
         validate_parser.add_argument(f'--validate-{action}', dest=action, action='store_true')
         validate_parser.add_argument(f'--no-validate-{action}', dest=action, action='store_false')
         validate_parser.set_defaults(**{action: True})
@@ -93,7 +93,7 @@ def main():
     elif args.action == 'clean':
         complete_cleanup()
     elif args.action == 'validate':
-        validate(args.system_state, args.manifest)
+        validate(args.system_state, args.manifest, args.datasets)
     elif args.action == 'branchout':
         validate_branch_out_config(not args.skip_push)
         branch_out_repos(not args.skip_push)

scale_build/validate.py

@@ -1,9 +1,11 @@
 import os
+import jsonschema
 import logging
 import shutil
 
 from .exceptions import CallError, MissingPackagesException
 from .utils.manifest import validate_manifest
+from truenas_install import fhs
 
 
 logger = logging.getLogger(__name__)
@@ -31,10 +33,21 @@ def validate_system_state():
         raise MissingPackagesException(missing_packages)
 
 
-def validate(system_state_flag=True, manifest_flag=True):
+def validate_datasets():
+    try:
+        jsonschema.validate(fhs.TRUENAS_DATASETS, fhs.TRUENAS_DATASET_SCHEMA)
+    except jsonschema.ValidationError as e:
+        raise CallError(f'Provided dataset schema is invalid: {e}')
+
+
+def validate(system_state_flag=True, manifest_flag=True, datasets_flag=True):
     if system_state_flag:
         validate_system_state()
         logger.debug('System state Validated')
     if manifest_flag:
         validate_manifest()
         logger.debug('Manifest Validated')
+
+    if datasets_flag:
+        validate_datasets()
+        logger.debug('Dataset schema Validated')

truenas_install/__main__.py

@@ -18,6 +18,7 @@ import textwrap
 import psutil
 
 from licenselib.license import ContractType, License
+from .fhs import TRUENAS_DATASETS
 
 logger = logging.getLogger(__name__)
 
@@ -389,6 +390,7 @@ def main():
                 if probe_dataset_name not in existing_datasets:
                     dataset_name = probe_dataset_name
                     break
+
     run_command([
         "zfs", "create",
         "-o", "mountpoint=legacy",
@@ -396,9 +398,40 @@ def main():
         "-o", "zectl:keep=False",
         dataset_name,
     ])
+
+    for entry in TRUENAS_DATASETS:
+        cmd = ["zfs", "create", "-u", "-o", "mountpoint=legacy", "-o", "canmount=noauto"]
+        if "NOSUID" in entry["options"]:
+            cmd.extend(["-o", "setuid=off", "-o", "devices=off"])
+        if "NOEXEC" in entry["options"]:
+            cmd.extend(["-o", "exec=off"])
+        if "NODEV" in entry["options"]:
+            cmd.extend(["-o", "devices=off"])
+        if "NOACL" in entry['options']:
+            cmd.extend(["-o", "acltype=off", "-o", "aclmode=discard"])
+        if "NOATIME" in entry["options"]:
+            cmd.extend(["-o", "atime=off"])
+
+        cmd.append(f"{dataset_name}/{entry['name']}")
+        run_command(cmd)
+
     try:
         with tempfile.TemporaryDirectory() as root:
+            undo = []
+            ds_info = []
             run_command(["mount", "-t", "zfs", dataset_name, root])
+            for entry in TRUENAS_DATASETS:
+                this_ds = entry['name']
+                ds_name = f"{dataset_name}/{this_ds}"
+                ds_path = entry.get("mountpoint") or f"/{entry['name']}"
+                ds_guid = run_command(["zfs", "list", "-o", "guid", "-H", ds_name]).stdout.strip()
+
+                mp = os.path.join(root, ds_path[1:])
+                os.makedirs(mp, exist_ok=True)
+                run_command(["mount", "-t", "zfs", f"{dataset_name}/{this_ds}", mp])
+                ds_info.append({"ds": ds_name, "guid": ds_guid, "fhs_entry": entry})
+                undo.append(["umount", mp])
+
             try:
                 write_progress(0, "Extracting")
                 cmd = [
@@ -424,7 +457,6 @@ def main():
                                 int(m.group("extracted")) / int(m.group("total")) * 0.5,
                                 "Extracting",
                             )
-
                             buffer = b""
 
                 p.wait()
@@ -434,6 +466,15 @@ def main():
 
                 write_progress(0.5, "Performing post-install tasks")
 
+                for entry in TRUENAS_DATASETS:
+                    if not (force_mode := entry.get("mode")):
+                        continue
+
+                    os.chmod(f"{root}/{entry['name']}", force_mode)
+
+                with open(f"{root}/conf/truenas_root_ds.json", "w") as f:
+                    f.write(json.dumps(ds_info, indent=4))
+
                 with contextlib.suppress(FileNotFoundError):
                     # We want to remove this for fresh installation + upgrade both
                     # In this case, /etc/machine-id would be treated as the valid
@@ -516,7 +557,6 @@ def main():
                 if IS_FREEBSD:
                     install_grub_freebsd(input, manifest, pool_name, dataset_name, disks)
                 else:
-                    undo = []
                     try:
                         run_command(["mount", "-t", "devtmpfs", "udev", f"{root}/dev"])
                         undo.append(["umount", f"{root}/dev"])
@@ -649,11 +689,24 @@ def main():
                             run_command(cmd)
             finally:
                 run_command(["umount", root])
+
+        for entry in TRUENAS_DATASETS:
+            this_ds = f"{dataset_name}/{entry['name']}"
+            mp = entry.get('mountpoint') or f"/{entry['name']}"
+            ro = "on" if "RO" in entry["options"] else "off"
+            run_command(["zfs", "set", f"readonly={ro}", this_ds])
+
+            if entry.get("snap", False):
+                run_command(["zfs", "snapshot", f"{this_ds}@pristine"])
+
+            run_command(["zfs", "set", f"mountpoint={mp}", this_ds])
+            run_command(["zfs", "set", 'org.zectl:bootloader=""', this_ds])
+
     except Exception:
         if old_bootfs_prop != "-":
             run_command(["zpool", "set", f"bootfs={old_bootfs_prop}", pool_name])
         if cleanup:
-            run_command(["zfs", "destroy", dataset_name])
+            run_command(["zfs", "destroy", "-r", dataset_name])
         raise
 
     configure_system_for_zectl(pool_name)

truenas_install/fhs.py

@@ -0,0 +1,144 @@
+"""
+TrueNAS Filesysten Heirarchy Standards extensions
+
+The TRUENAS_DATASET dictionary contains the dataset configuration
+settings for the root filesystem of the truenas server.
+
+When practical it is best to turn off unnecessary features.
+For example, disabling ACL support on system files simplifies
+permissions auditing. Disabling ACL support also ensures compliance
+with STIGs that require configuration files and libraries to not have ACLs
+present.
+
+KEYS
+--------------
+The following keys are supported:
+`name` - the name of the dataset (will be appended to other dataset name
+    related components
+`options` - Dataset configuration options (explained below). There is no
+    default.
+`mode` - permissions to set on the dataset's mountpoint during installation
+    default is 0o755
+`mountpoint` - dataset mountpoint. If no mountpoint is specified then it
+    /`name` will be assumed.
+`snap` - Take a snapshot named "pristine" after creating the dataset.
+    default is False
+
+OPTIONS
+--------------
+NOSUID - sets a combination of setuid=off and devices=off
+NOEXEC - sets exec=off
+NOACL - sets acltype=off.
+NOATIME - sets atime=off.
+RO - sets readonly=on.
+NODEV - sets devices=off
+
+DATASETS
+--------------
+audit - dataset used for storing system auditing databases
+conf - truenas configuration files - static
+data - truenas configuraiton files - dynamic
+etc - see FHS
+home - see FHS, NOTE: only `admin` account will be present
+mnt - TrueNAS does not follow FHS for this path. It is reserved exclusively
+    for ZFS pool mountpoints.
+opt - see FHS
+usr - see FHS
+var - see FHS
+var/log - separate dataset for system logs. This is to provide flexibility to
+    snapshot and replicate log information as needed by administrator.
+var/ca-certificates - administrator-provided CA certificates, symlinked
+    from /usr/local/share/ca-certificates.
+"""
+
+
+# Following schema is used for validation (e.g. "make validate") in scale-build
+# If any changes are made to OPTIONS or KEYS above then schema must be updated
+# accordingly.
+TRUENAS_DATASET_SCHEMA = {
+    'type': 'array',
+    'items': {
+        'type': 'object',
+        'properties': {
+            'name': {'type': 'string'},
+            'options': {
+                'type': 'array',
+                'items': {
+                    'type': 'string',
+                    'enum': [
+                        'NOSUID',
+                        'NOEXEC',
+                        'NOACL',
+                        'NOATIME',
+                        'RO',
+                        'NODEV',
+                    ]
+                },
+                'uniqueItems': True,
+            },
+            'mode': {'type': 'integer'},
+            'mountpoint': {'type': 'string'},
+            'snap': {'type': 'boolean'},
+        },
+        'required': ['name', 'options'],
+        'additionalProperties': False,
+    }
+}
+
+
+TRUENAS_DATASETS = [
+    {
+        'name':  'audit',
+        'options': ['NOSUID', 'NOEXEC', 'NOATIME', 'NOACL'],
+        'mode': 0o700
+    },
+    {
+        'name':  'conf',
+        'options': ['NOSUID', 'NOEXEC', 'RO', 'NOACL'],
+        'mode': 0o700,
+        'snap': True
+    },
+    {
+        'name':  'data',
+        'options': ['NOSUID', 'NOEXEC', 'NOACL', 'NOATIME'],
+        'mode': 0o700,
+        'snap': True
+    },
+    {
+        'name':  'mnt',
+        'options': ['NOSUID', 'NOEXEC', 'NOACL', 'NOATIME'],
+    },
+    {
+        'name':  'etc',
+        'options': ['NOSUID', 'NOACL', 'NOEXEC'],
+        'snap': True
+    },
+    {
+        'name':  'home',
+        'options': ['NOSUID', 'NOACL', 'NOEXEC']
+    },
+    {
+        'name':  'opt',
+        'options': ['NOSUID', 'NOACL'],
+        'snap': True
+    },
+    {
+        'name':  'usr',
+        'options': ['NOSUID', 'NOACL', 'RO', 'NOATIME'],
+        'snap': True
+    },
+    {
+        'name':  'var',
+        'options': ['NOSUID', 'NOACL', 'NOATIME'],
+        'snap': True
+    },
+    {
+        'name':  'var/ca-certificates',
+        'options': ['NOSUID', 'NOACL', 'NOEXEC'],
+        'mountpoint': '/var/local/ca-certificates'
+    },
+    {
+        'name':  'var/log',
+        'options': ['NOSUID', 'NOEXEC', 'NOACL', 'NOATIME']
+    },
+]