From 6fe49f046996f03d8328d24003f68565ca30c014 Mon Sep 17 00:00:00 2001 From: Isaac Hermida Date: Tue, 7 Apr 2026 13:26:42 +0200 Subject: [PATCH] containers: create external script to generate DCP https://onedigi.atlassian.net/browse/DEL-10037 Signed-off-by: Isaac Hermida --- meta-digi-containers/README.md | 70 +++- .../examples/manifest-lxc.json | 20 + .../examples/manifest-podman.json | 21 ++ meta-digi-containers/scripts/generate-dcp.py | 342 ++++++++++++++++++ 4 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 meta-digi-containers/examples/manifest-lxc.json create mode 100644 meta-digi-containers/examples/manifest-podman.json create mode 100755 meta-digi-containers/scripts/generate-dcp.py diff --git a/meta-digi-containers/README.md b/meta-digi-containers/README.md index 1321c734f..f47c2784d 100644 --- a/meta-digi-containers/README.md +++ b/meta-digi-containers/README.md @@ -31,6 +31,66 @@ Note: Podman archive generation requires an OCI image output as intermediate inp The recipe keeps `oci` in `IMAGE_FSTYPES` because `do_image_podman_archive` converts that OCI artifact into a `docker-archive` tar using `skopeo`. +## External Digi Container Package (DCP) generation + +Use the following Python script to generate a DCP out of Yocto workspace: + +- `meta-digi-containers/scripts/generate-dcp.py` + +This script requires: + +- `manifest.json` +- payload artifact, which may be one of: + - Podman: `image.tar` + - LXC: a Yocto-style LXC bundle (`.tar.xz`) containing `rootfs/` and `config` + +Usage: + +```bash +python3 meta-digi-containers/scripts/generate-dcp.py \ + --manifest /path/to/manifest.json \ + --payload /path/to/payload \ + [--output-dir /path/to/outdir] \ + [--readme /path/to/README.txt] \ + [--changelog /path/to/changelog.txt] +``` + +and generates a final DCP bundle with the same layout used today by the Yocto recipe: + +- `manifest.json` +- `payload/` +- `checksums/sha256sums.txt` +- `metadata/README.txt` +- `metadata/changelog.txt` + +The script generates exactly one runtime artifact per execution. + +Notes: + +- `--output-dir` is optional. If omitted, the script writes the DCP to the current directory. +- The output file name is always derived from manifest fields as: + - `_artifact__.tar.gz` +- For Podman payloads, the final internal payload name is always `payload/image.tar` +- For LXC payloads, the generator requires a Yocto-style `.tar.gz` bundle, validates its layout, + and stores it inside the DCP using the original file name and format without re-packaging it. + +Example manifests are provided at: + +- `meta-digi-containers/examples/manifest-lxc.json` +- `meta-digi-containers/examples/manifest-podman.json` + +Example: + +```bash +python3 meta-digi-containers/scripts/generate-dcp.py \ + --manifest meta-digi-containers/examples/manifest-podman.json \ + --payload /tmp/image.tar +``` + +This generates: + +- `./flutter-demo_artifact_podman_ccmp25-dvk.tar.gz` + ## Layer Scope Main recipes: @@ -91,8 +151,14 @@ Outputs are generated in: Final outputs: -- `${CONTAINER_NAME}_artifact_podman_${MACHINE}.tar.gz` -- `${CONTAINER_NAME}_artifact_lxc_${MACHINE}.tar.gz` +- `${CONTAINER_PACKAGE_ID}_artifact_podman_.tar.gz` +- `${CONTAINER_PACKAGE_ID}_artifact_lxc_.tar.gz` + +Notes: + +- The generator script makes the DCP filename using `package_id`, `runtime`, and + `device_types[0]` fields from the manifest file. +- In Yocto builds, `CONTAINER_PACKAGE_ID` defaults to `${CONTAINER_NAME}` Intermediate outputs generated during the build (LXC bundle, Podman archive, OCI/rootfs files) are removed automatically at the end of artifact creation. diff --git a/meta-digi-containers/examples/manifest-lxc.json b/meta-digi-containers/examples/manifest-lxc.json new file mode 100644 index 000000000..ab59a5a9e --- /dev/null +++ b/meta-digi-containers/examples/manifest-lxc.json @@ -0,0 +1,20 @@ +{ + "package_id": "flutter-demo", + "version": "1.0", + "runtime": "lxc", + "registration_defaults": { + "autostart": true, + "monitor": true, + "restart": { + "enabled": true, + "max_retries": 3, + "window": 60, + "retry_delay": 3 + } + }, + "device_types": ["ccmp25-dvk"], + "firmware_versions": "", + "build_id": "", + "description": "", + "labels": {} +} diff --git a/meta-digi-containers/examples/manifest-podman.json b/meta-digi-containers/examples/manifest-podman.json new file mode 100644 index 000000000..9a8c18fcc --- /dev/null +++ b/meta-digi-containers/examples/manifest-podman.json @@ -0,0 +1,21 @@ +{ + "package_id": "flutter-demo", + "version": "1.0", + "runtime": "podman", + "create_args": "--privileged --network none --tmpfs /dev/shm:rw,nosuid,nodev,mode=1777 --device /dev/dri --device /dev/input --device /dev/galcore --device /dev/tty --device /dev/tty0 --device /dev/tty1 --device /dev/tty7 --volume /run/udev:/run/udev:ro --tty --entrypoint '[\"/usr/bin/docker-init\",\"/start-flutter-demo.sh\"]'", + "registration_defaults": { + "autostart": true, + "monitor": true, + "restart": { + "enabled": true, + "max_retries": 3, + "window": 60, + "retry_delay": 3 + } + }, + "device_types": ["ccmp25-dvk"], + "firmware_versions": "", + "build_id": "", + "description": "", + "labels": {} +} diff --git a/meta-digi-containers/scripts/generate-dcp.py b/meta-digi-containers/scripts/generate-dcp.py new file mode 100755 index 000000000..999a8d1b7 --- /dev/null +++ b/meta-digi-containers/scripts/generate-dcp.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026, Digi International Inc. +# +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +import os +from pathlib import Path +import shutil +import sys +import tarfile +import tempfile + + +def fail(message: str) -> "NoReturn": + raise SystemExit(f"error: {message}") + + +def load_manifest(path: Path) -> dict: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + fail(f"manifest file not found: {path}") + except json.JSONDecodeError as exc: + fail(f"invalid manifest JSON: {exc}") + except OSError as exc: + fail(f"cannot read manifest file {path}: {exc}") + if not isinstance(data, dict): + fail("manifest must be a JSON object") + return data + + +def validate_bool(container: dict, key: str, *, path: str) -> bool: + value = container.get(key) + if not isinstance(value, bool): + fail(f"invalid manifest: {path}.{key} must be boolean") + return value + + +def validate_int(container: dict, key: str, *, path: str, minimum: int) -> int: + value = container.get(key) + if isinstance(value, bool) or not isinstance(value, int): + fail(f"invalid manifest: {path}.{key} must be integer") + if value < minimum: + fail(f"invalid manifest: {path}.{key} must be >= {minimum}") + return value + + +def validate_string(container: dict, key: str, *, path: str, required: bool = True) -> str: + value = container.get(key) + if value is None and not required: + return "" + if not isinstance(value, str) or not value.strip(): + fail(f"invalid manifest: {path}.{key} must be a non-empty string") + return value.strip() + + +def validate_device_types(value: object) -> list[str]: + if not isinstance(value, list) or not value: + fail("invalid manifest: device_types must be a non-empty list") + result: list[str] = [] + for item in value: + if not isinstance(item, str) or not item.strip(): + fail("invalid manifest: device_types entries must be non-empty strings") + result.append(item.strip()) + return result + + +def validate_labels(value: object) -> dict: + if value is None: + return {} + if not isinstance(value, dict): + fail("invalid manifest: labels must be a JSON object") + return value + + +def validate_manifest(data: dict) -> dict: + package_id = validate_string(data, "package_id", path="manifest") + version = validate_string(data, "version", path="manifest") + runtime = validate_string(data, "runtime", path="manifest") + if runtime not in {"lxc", "podman"}: + fail("invalid manifest: runtime must be one of: lxc, podman") + device_types = validate_device_types(data.get("device_types")) + + registration_defaults = data.get("registration_defaults") + if not isinstance(registration_defaults, dict): + fail("invalid manifest: registration_defaults must be an object") + restart = registration_defaults.get("restart") + if not isinstance(restart, dict): + fail("invalid manifest: registration_defaults.restart must be an object") + + create_args = data.get("create_args") + if runtime == "podman": + if not isinstance(create_args, str) or not create_args.strip(): + fail("invalid manifest: create_args is required for podman") + create_args = create_args.strip() + elif create_args is not None: + fail("invalid manifest: create_args is not allowed for lxc") + + firmware_versions = data.get("firmware_versions", "") + if firmware_versions is not None and not isinstance(firmware_versions, str): + fail("invalid manifest: firmware_versions must be a string") + + build_id = data.get("build_id", "") + if build_id is not None and not isinstance(build_id, str): + fail("invalid manifest: build_id must be a string") + + description = data.get("description", "") + if description is not None and not isinstance(description, str): + fail("invalid manifest: description must be a string") + + return { + "package_id": package_id, + "version": version, + "runtime": runtime, + "create_args": create_args, + "registration_defaults": { + "autostart": validate_bool(registration_defaults, "autostart", path="registration_defaults"), + "monitor": validate_bool(registration_defaults, "monitor", path="registration_defaults"), + "restart": { + "enabled": validate_bool(restart, "enabled", path="registration_defaults.restart"), + "max_retries": validate_int(restart, "max_retries", path="registration_defaults.restart", minimum=0), + "window": validate_int(restart, "window", path="registration_defaults.restart", minimum=1), + "retry_delay": validate_int(restart, "retry_delay", path="registration_defaults.restart", minimum=0), + }, + }, + "device_types": device_types, + "firmware_versions": firmware_versions or "", + "build_id": build_id or "", + "description": description or "", + "labels": validate_labels(data.get("labels")), + } + + +def validate_payload(path: Path, runtime: str) -> None: + if not path.exists(): + fail(f"payload file not found: {path}") + if not path.is_file(): + fail(f"payload path is not a file: {path}") + lowered = path.name.lower() + if runtime == "podman": + if not lowered.endswith(".tar"): + fail("invalid payload: runtime podman requires a .tar file") + return + if not lowered.endswith(".tar.xz"): + fail("invalid payload: runtime lxc requires a .tar.xz file") + + +def tar_mode_for_path(path: Path, *, writing: bool = False) -> str: + lowered = path.name.lower() + if lowered.endswith((".tgz", ".tar.gz")): + return "w:gz" if writing else "r:gz" + if lowered.endswith(".tar.xz"): + return "w:xz" if writing else "r:xz" + return "w:" if writing else "r:" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_output_name(manifest: dict) -> str: + return f"{manifest['package_id']}_artifact_{manifest['runtime']}_{manifest['device_types'][0]}.tar.gz" + + +def ensure_lxc_layout(payload: Path) -> tuple[str, str]: + with tempfile.TemporaryDirectory(prefix="dcp-lxc-validate-") as tmpdir: + root = Path(tmpdir) + try: + with tarfile.open(payload, tar_mode_for_path(payload)) as tar: + tar.extractall(root) + except (tarfile.TarError, OSError) as exc: + fail(f"invalid payload: cannot unpack LXC archive {payload}: {exc}") + + if (root / "rootfs").is_dir() and (root / "config").is_file(): + return "", "" + + dirs = [item for item in root.iterdir() if item.is_dir()] + if len(dirs) == 1 and (dirs[0] / "rootfs").is_dir() and (dirs[0] / "config").is_file(): + return dirs[0].name, dirs[0].name + + fail("invalid payload: LXC archive must contain rootfs/ and config") + + +def write_default_readme(path: Path, manifest: dict, *, created_at: str, payload_name: str) -> None: + content = ( + f"Package: {manifest['package_id']}\n" + f"Version: {manifest['version']}\n" + f"Runtime: {manifest['runtime']}\n" + f"Platform: {manifest['device_types'][0]}\n" + f"Payload: {payload_name}\n" + f"Generated: {created_at}\n" + ) + path.write_text(content, encoding="utf-8") + + +def write_default_changelog(path: Path) -> None: + path.write_text("Initial package generated with external DCP tool.\n", encoding="utf-8") + + +def stage_metadata(src: Path | None, dst: Path, default_writer) -> None: + if src is not None: + try: + shutil.copy2(src, dst) + except OSError as exc: + fail(f"cannot copy metadata file {src} to {dst}: {exc}") + else: + default_writer(dst) + + +def write_manifest(path: Path, manifest: dict) -> None: + path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + +def build_final_manifest(base: dict, *, artifact_type: str, digest: str, size_bytes: int, created_at: str) -> dict: + output = { + "package_id": base["package_id"], + "version": base["version"], + "runtime": base["runtime"], + "artifact_type": artifact_type, + "registration_defaults": base["registration_defaults"], + "created_at": created_at, + "digest": f"sha256:{digest}", + "device_types": base["device_types"], + "firmware_versions": base["firmware_versions"], + "size_bytes": size_bytes, + "build_id": base["build_id"], + "description": base["description"], + "labels": base["labels"], + } + if base["runtime"] == "podman": + output["create_args"] = base["create_args"] + return output + + +def pack_output(staging_root: Path, output_path: Path) -> None: + try: + with tarfile.open(output_path, "w:gz") as tar: + for item in sorted(staging_root.iterdir()): + tar.add(item, arcname=item.name) + except OSError as exc: + fail(f"cannot create output archive {output_path}: {exc}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate a Digi container DCP bundle from a manifest and a single payload artifact." + ) + parser.add_argument("--manifest", required=True, help="Path to input manifest.json") + parser.add_argument("--payload", required=True, help="Path to input payload file") + parser.add_argument( + "--output-dir", + default=".", + help="Optional output directory. Defaults to the current working directory.", + ) + parser.add_argument("--readme", help="Optional README.txt to include under metadata/") + parser.add_argument("--changelog", help="Optional changelog.txt to include under metadata/") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + manifest_path = Path(args.manifest) + payload_path = Path(args.payload) + output_dir = Path(args.output_dir) + readme_path = Path(args.readme) if args.readme else None + changelog_path = Path(args.changelog) if args.changelog else None + + manifest = validate_manifest(load_manifest(manifest_path)) + validate_payload(payload_path, manifest["runtime"]) + + try: + output_dir.mkdir(parents=True, exist_ok=True) + except OSError as exc: + fail(f"cannot create output directory {output_dir}: {exc}") + + output_name = build_output_name(manifest) + output_path = output_dir / output_name + created_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + with tempfile.TemporaryDirectory(prefix="dcp-bundle-") as tmpdir: + staging_root = Path(tmpdir) + payload_dir = staging_root / "payload" + checksums_dir = staging_root / "checksums" + metadata_dir = staging_root / "metadata" + payload_dir.mkdir(parents=True, exist_ok=True) + checksums_dir.mkdir(parents=True, exist_ok=True) + metadata_dir.mkdir(parents=True, exist_ok=True) + + if manifest["runtime"] == "podman": + payload_name = "image.tar" + artifact_type = "podman-image-tar" + staged_payload = payload_dir / payload_name + shutil.copy2(payload_path, staged_payload) + else: + ensure_lxc_layout(payload_path) + payload_name = payload_path.name + staged_payload = payload_dir / payload_name + artifact_type = "lxc-bundle" + shutil.copy2(payload_path, staged_payload) + + digest = sha256(staged_payload) + size_bytes = staged_payload.stat().st_size + + final_manifest = build_final_manifest( + manifest, + artifact_type=artifact_type, + digest=digest, + size_bytes=size_bytes, + created_at=created_at, + ) + write_manifest(staging_root / "manifest.json", final_manifest) + + stage_metadata( + readme_path, + metadata_dir / "README.txt", + lambda dst: write_default_readme(dst, manifest, created_at=created_at, payload_name=payload_name), + ) + stage_metadata(changelog_path, metadata_dir / "changelog.txt", write_default_changelog) + + (checksums_dir / "sha256sums.txt").write_text( + f"{digest} payload/{payload_name}\n", + encoding="utf-8", + ) + + pack_output(staging_root, output_path) + + print(output_path.resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())