diff --git a/builder/bash/common.sh b/builder/bash/common.sh index b6be69b..320c1d6 100755 --- a/builder/bash/common.sh +++ b/builder/bash/common.sh @@ -21,14 +21,3 @@ load_env_file() { set +a } -maybe_fake_legacy() { - TOOL_NAME="$1" - if command -v "$TOOL_NAME" >/dev/null 2>&1; then - return 1 - fi - if [ "${RETRODEBIAN_FAKE_LEGACY_TOOLS:-1}" = "1" ]; then - echo "[fake-legacy] $TOOL_NAME not found, emulating." >&2 - return 0 - fi - fail "$TOOL_NAME not found and RETRODEBIAN_FAKE_LEGACY_TOOLS != 1" -} diff --git a/builder/bash/run_profile_build.sh b/builder/bash/run_profile_build.sh index 04edf3f..2ebfddd 100755 --- a/builder/bash/run_profile_build.sh +++ b/builder/bash/run_profile_build.sh @@ -12,11 +12,4 @@ load_env_file "$ENV_FILE" [ -n "${PROFILE_ARTIFACT_DIR:-}" ] || fail "PROFILE_ARTIFACT_DIR is required" mkdir -p "$PROFILE_ARTIFACT_DIR/final" -if maybe_fake_legacy lh_build; then - OUTPUT_FILE="$PROFILE_ARTIFACT_DIR/final/${PROFILE_NAME:-profile}.iso" - printf 'fake iso for %s -' "${PROFILE_NAME:-unknown}" > "$OUTPUT_FILE" - exit 0 -fi - exec lh_build "$LIVE_DIR" diff --git a/builder/bash/run_profile_config.sh b/builder/bash/run_profile_config.sh index 10eb38c..42fdeac 100755 --- a/builder/bash/run_profile_config.sh +++ b/builder/bash/run_profile_config.sh @@ -11,11 +11,10 @@ load_env_file "$ENV_FILE" [ -n "${LIVE_DIR:-}" ] || fail "LIVE_DIR is required" mkdir -p "$LIVE_DIR" -if maybe_fake_legacy lh_config; then - mkdir -p "$LIVE_DIR/.fake-live-helper" - printf 'fake lh_config for %s -' "${PROFILE_NAME:-unknown}" > "$LIVE_DIR/.fake-live-helper/lh_config.txt" - exit 0 -fi - -exec lh_config "$LIVE_DIR" +exec lh_config \ + --mirror-bootstrap $(BASE__REPO_URL) \ + --mirror-binary $(BASE__REPO_URL) \ + --mirror-binary-security $(BASE__SECURITY_REPO_URL) \ + --mirror-chroot $(BASE__REPO_URL) \ + --mirror-chroot-security $(BASE__SECURITY_REPO_URL) \ + "$LIVE_DIR" diff --git a/builder/py/orchestrate.py b/builder/py/orchestrate.py index b36f79e..28c663d 100644 --- a/builder/py/orchestrate.py +++ b/builder/py/orchestrate.py @@ -16,7 +16,14 @@ if str(THIS_DIR) not in sys.path: from retrobuilder.context import BuildContext from retrobuilder.envfile import write_env_file -from retrobuilder.loader import load_base_chain, load_base_entry, load_base_spec, load_feature_entry, load_feature_spec, load_profile +from retrobuilder.loader import ( + load_base_chain, + load_module_entry, + load_module_spec, + load_profile, + resolve_module_config, + resolve_module_docker_override, +) from retrobuilder.operations import ( apply_profile_common_configuration, clear_directory_contents, @@ -39,7 +46,7 @@ DEFAULT_RUNTIME_DOCKERFILES = { } -def run(cmd: list[str], *, cwd: Path | None = None, capture: bool = False) -> str: +def sh(cmd: list[str], *, cwd: Path | None = None, capture: bool = False) -> str: proc = subprocess.run(cmd, cwd=cwd, text=True, check=True, stdout=subprocess.PIPE if capture else None) return proc.stdout.strip() if capture else "" @@ -64,8 +71,8 @@ class RuntimeConfig: class DockerRuntime: - def __init__(self, project_root: Path, name: str, image: str) -> None: - self.project_root = project_root + def __init__(self, root: Path, name: str, image: str) -> None: + self.root = root self.name = name self.image = image self.container = f"retrodebian-{name}-{uuid.uuid4().hex[:8]}" @@ -75,171 +82,126 @@ class DockerRuntime: if self.started: return user = f"{os.getuid()}:{os.getgid()}" if hasattr(os, "getuid") else "0:0" - run([ + sh([ "docker", "run", "-d", "--name", self.container, "--user", user, - "-w", str(self.project_root), - "-v", f"{self.project_root}:{self.project_root}:rw", - self.image, - "sleep", "infinity", + "-w", str(self.root), + "-v", f"{self.root}:{self.root}:rw", + self.image, "sleep", "infinity", ]) self.started = True def exec(self, argv: list[str], env: dict[str, str] | None = None) -> None: self.start() cmd = ["docker", "exec"] - for key, value in (env or {}).items(): - cmd.extend(["-e", f"{key}={value}"]) - cmd.append(self.container) - cmd.extend(argv) - run(cmd) + for k, v in (env or {}).items(): + cmd += ["-e", f"{k}={v}"] + sh(cmd + [self.container] + argv) def stop(self) -> None: if self.started: try: - run(["docker", "stop", self.container]) + sh(["docker", "stop", self.container]) finally: self.started = False def rm(self) -> None: - run(["docker", "rm", "-f", self.container]) + sh(["docker", "rm", "-f", self.container]) class RuntimePool: def __init__(self, root: Path, configs: dict[str, RuntimeConfig], keep: bool = False) -> None: self.root = root self.keep = keep - self.configs = configs - self.defaults = {name: DockerRuntime(root, name, self._resolve_image(name, cfg)) for name, cfg in configs.items()} + self.defaults = {name: DockerRuntime(root, name, self._image(name, cfg)) for name, cfg in configs.items()} - def _resolve_image(self, name: str, cfg: RuntimeConfig) -> str: + def _image(self, name: str, cfg: RuntimeConfig) -> str: if cfg.dockerfile: image = cfg.image or f"retrodebian/{name}:local" - run(["docker", "build", "-t", image, "-f", cfg.dockerfile, cfg.context or "."], cwd=self.root) + sh(["docker", "build", "-t", image, "-f", cfg.dockerfile, cfg.context or "."], cwd=self.root) return image if cfg.image: return cfg.image - raise ValueError(f"Missing runtime image for {name}") + raise ValueError(f"Missing image for runtime {name}") - def start_defaults(self) -> None: + def start(self) -> None: for runtime in self.defaults.values(): runtime.start() def close(self) -> None: for runtime in self.defaults.values(): - try: - runtime.stop() - except Exception: - pass + try: runtime.stop() + except Exception: pass if not self.keep: for runtime in self.defaults.values(): - try: - runtime.rm() - except Exception: - pass + try: runtime.rm() + except Exception: pass - def get(self, runtime_name: str, override: RuntimeConfig | None = None) -> DockerRuntime: + def resolve(self, runtime_name: str, override: RuntimeConfig | None = None) -> tuple[DockerRuntime, bool]: if not override or not (override.image or override.dockerfile): - return self.defaults[runtime_name] - image = self._resolve_image(f"{runtime_name}-override-{uuid.uuid4().hex[:6]}", override) - return DockerRuntime(self.root, f"{runtime_name}-override", image) - - -def ctx(root: Path, phase: str, kind: str, profile_name: str = "", base_name: str = "", feature_name: str = "") -> BuildContext: - profile = load_profile(root, profile_name) if profile_name else None - current_name = feature_name or base_name or profile_name - return BuildContext( - project_root=root, - live_dir=root / "live", - artifacts_root=root / "artifacts", - phase=phase, - current_kind=kind, - current_name=current_name, - current_module_artifact_dir=( - feature_artifacts_dir(root, feature_name) if kind == "feature" else - base_artifacts_dir(root, base_name) if kind == "base" else - profile_artifacts_dir(root, profile_name) - ), - profile_name=profile_name, - profile_artifact_dir=profile_artifacts_dir(root, profile_name) if profile_name else None, - base_name=base_name, - base_artifact_dir=base_artifacts_dir(root, base_name) if base_name else None, - feature_name=feature_name, - feature_artifact_dir=feature_artifacts_dir(root, feature_name) if feature_name else None, - profile_features=tuple(profile.features) if profile else (), - ) - - -def env_for(context: BuildContext, fake_legacy: bool) -> dict[str, str]: - env = context.to_env() - env["RETRODEBIAN_FAKE_LEGACY_TOOLS"] = "1" if fake_legacy else os.environ.get("RETRODEBIAN_FAKE_LEGACY_TOOLS", "0") - return env - - -def write_phase_env(context: BuildContext, path: Path, fake_legacy: bool) -> Path: - write_env_file(path, env_for(context, fake_legacy)) - return path - - -def load_module(kind: str, root: Path, name: str): - return (load_base_spec(root, name), load_base_entry(root, name), base_dir(root, name)) if kind == "base" else (load_feature_spec(root, name), load_feature_entry(root, name), feature_dir(root, name)) - - -def override_for(spec, phase: str) -> RuntimeConfig | None: - stage = spec.docker_overrides.get(phase) - if not stage: - return None - return RuntimeConfig(stage.image or "", stage.dockerfile or "", stage.docker_context or ".") + return self.defaults[runtime_name], False + runtime = DockerRuntime(self.root, f"{runtime_name}-override", self._image(f"{runtime_name}-override-{uuid.uuid4().hex[:6]}", override)) + runtime.start() + return runtime, True class Orchestrator: - def __init__(self, root: Path, runtimes: RuntimePool, fake_legacy: bool = False) -> None: + def __init__(self, root: Path, runtimes: RuntimePool) -> None: self.root = root self.runtimes = runtimes - self.fake_legacy = fake_legacy def validate(self) -> None: for profile_name in list_names(self.root / "profiles"): profile = load_profile(self.root, profile_name) load_base_chain(self.root, profile.base) for feature_name in profile.features: - load_feature_spec(self.root, feature_name) + load_module_spec(self.root, "feature", feature_name) print("Validation OK") - def python_phase(self, kind: str, name: str, phase: str, *, profile: str = "", base: str = "", feature: str = "") -> None: - spec, _entry, module_dir = load_module(kind, self.root, name) - runtime = self.runtimes.get(ORCHESTRATOR_RUNTIME, override_for(spec, phase)) - owned = runtime is not self.runtimes.defaults[ORCHESTRATOR_RUNTIME] - if owned: - runtime.start() + def context(self, phase: str, kind: str, *, profile_name: str = "", base_name: str = "", feature_name: str = "") -> BuildContext: + profile = load_profile(self.root, profile_name) if profile_name else None + base_name = base_name or (profile.base if profile else "") + feature_names = tuple(profile.features) if profile else () + return BuildContext( + project_root=self.root, + live_dir=self.root / "live", + artifacts_root=self.root / "artifacts", + phase=phase, + kind=kind, + name=feature_name or base_name or profile_name, + profile_name=profile_name, + base_name=base_name, + feature_name=feature_name, + profile={**(profile.config if profile else {}), "base": base_name, "features": feature_names, "edition": profile.edition if profile else "", "description": profile.description if profile else "", "splash": profile.splash if profile else ""}, + base=resolve_module_config(self.root, "base", base_name) if base_name else {}, + features={name: resolve_module_config(self.root, "feature", name) for name in feature_names}, + ) + + def env_file(self, ctx: BuildContext, path: Path) -> Path: + values = ctx.to_env() + write_env_file(path, values) + return path + + def override(self, kind: str, name: str, phase: str) -> RuntimeConfig | None: + stage = resolve_module_docker_override(self.root, kind, name, phase) + if not stage: + return None + return RuntimeConfig(stage.image, stage.dockerfile, stage.docker_context or ".") + + def run_python(self, kind: str, name: str, phase: str, *, profile_name: str = "", base_name: str = "", feature_name: str = "") -> None: + module_dir = (base_dir if kind == "base" else feature_dir)(self.root, name) + runtime, owned = self.runtimes.resolve(ORCHESTRATOR_RUNTIME, self.override(kind, name, phase)) try: - runtime.exec(["python3", str(module_dir / "entry.py"), phase], env_for(ctx(self.root, phase, kind, profile, base, feature), self.fake_legacy)) + runtime.exec(["python3", str(module_dir / "entry.py"), phase], self.context(phase, kind, profile_name=profile_name, base_name=base_name, feature_name=feature_name).to_env()) finally: if owned: runtime.stop() if not self.runtimes.keep: runtime.rm() - def shell_phase(self, kind: str, name: str, phase: str, env_path: Path) -> None: - spec, _entry, module_dir = load_module(kind, self.root, name) - runtime = self.runtimes.get(LIVE_HELPER_RUNTIME, override_for(spec, phase)) - owned = runtime is not self.runtimes.defaults[LIVE_HELPER_RUNTIME] - if owned: - runtime.start() - try: - runtime.exec(["builder/bash/run_entry.sh", str(module_dir), str(env_path), phase]) - finally: - if owned: - runtime.stop() - if not self.runtimes.keep: - runtime.rm() - - def generate(self, kind: str, name: str, env_path: Path) -> None: - spec, _entry, module_dir = load_module(kind, self.root, name) - runtime = self.runtimes.get(PACKAGE_BUILDER_RUNTIME, override_for(spec, "generate")) - owned = runtime is not self.runtimes.defaults[PACKAGE_BUILDER_RUNTIME] - if owned: - runtime.start() + def run_generate(self, kind: str, name: str, env_path: Path) -> None: + module_dir = (base_dir if kind == "base" else feature_dir)(self.root, name) + runtime, owned = self.runtimes.resolve(PACKAGE_BUILDER_RUNTIME, self.override(kind, name, "generate")) try: runtime.exec(["builder/bash/run_generate.sh", str(module_dir), str(env_path)]) finally: @@ -248,63 +210,65 @@ class Orchestrator: if not self.runtimes.keep: runtime.rm() - def run_profile_config(self, profile: str, env_path: Path) -> None: - self.runtimes.defaults[LIVE_HELPER_RUNTIME].exec(["builder/bash/run_profile_config.sh", str(env_path)]) - - def run_profile_build(self, profile: str, env_path: Path) -> None: - self.runtimes.defaults[LIVE_HELPER_RUNTIME].exec(["builder/bash/run_profile_build.sh", str(env_path)]) + def run_shell(self, kind: str, name: str, phase: str, env_path: Path) -> None: + module_dir = (base_dir if kind == "base" else feature_dir)(self.root, name) + runtime, owned = self.runtimes.resolve(LIVE_HELPER_RUNTIME, self.override(kind, name, phase)) + try: + runtime.exec(["builder/bash/run_entry.sh", str(module_dir), str(env_path), phase]) + finally: + if owned: + runtime.stop() + if not self.runtimes.keep: + runtime.rm() def build_common_features(self, profile_hint: str) -> None: - for name in list_names(self.root / "features"): - pre = ctx(self.root, "pre-gen", "feature", profile_hint, feature_name=name) - self.python_phase("feature", name, "pre-gen", profile=profile_hint, feature=name) - self.generate("feature", name, write_phase_env(pre, feature_artifacts_dir(self.root, name) / "runtime.env", self.fake_legacy)) - self.python_phase("feature", name, "post-gen", profile=profile_hint, feature=name) + for feature_name in list_names(self.root / "features"): + ctx = self.context("pre-gen", "feature", profile_name=profile_hint, feature_name=feature_name) + self.run_python("feature", feature_name, "pre-gen", profile_name=profile_hint, feature_name=feature_name) + self.run_generate("feature", feature_name, self.env_file(ctx, feature_artifacts_dir(self.root, feature_name) / "runtime.env")) + self.run_python("feature", feature_name, "post-gen", profile_name=profile_hint, feature_name=feature_name) - def build_base_chain(self, profile_name: str, chain: list[tuple[str, object]]) -> None: - for name, _spec in chain: - pre = ctx(self.root, "pre-gen", "base", profile_name, base_name=name) - self.python_phase("base", name, "pre-gen", profile=profile_name, base=name) - self.generate("base", name, write_phase_env(pre, base_artifacts_dir(self.root, name) / "runtime.env", self.fake_legacy)) - self.python_phase("base", name, "post-gen", profile=profile_name, base=name) + def build_base_chain(self, profile_name: str, base_chain: list[tuple[str, object]]) -> None: + for base_name, _ in base_chain: + ctx = self.context("pre-gen", "base", profile_name=profile_name, base_name=base_name) + self.run_python("base", base_name, "pre-gen", profile_name=profile_name, base_name=base_name) + self.run_generate("base", base_name, self.env_file(ctx, base_artifacts_dir(self.root, base_name) / "runtime.env")) + self.run_python("base", base_name, "post-gen", profile_name=profile_name, base_name=base_name) def configure_profile(self, profile_name: str, profile, base_chain: list[tuple[str, object]]) -> None: - live_dir = self.root / "live" - clear_directory_contents(live_dir) - ensure_live_structure(live_dir) - save_profile_metadata(profile_artifacts_dir(self.root, profile_name) / "profile.json", profile_name, profile, profile.base, load_base_spec(self.root, profile.base), base_chain) - config_ctx = ctx(self.root, "config", "profile", profile_name) - self.run_profile_config(profile_name, write_phase_env(config_ctx, profile_artifacts_dir(self.root, profile_name) / "profile-config.env", self.fake_legacy)) - apply_profile_common_configuration(self.root, live_dir, profile_name, profile) + clear_directory_contents(self.root / "live") + ensure_live_structure(self.root / "live") + save_profile_metadata(profile_artifacts_dir(self.root, profile_name) / "profile.json", profile_name, profile, profile.base, load_module_spec(self.root, "base", profile.base), base_chain) + self.runtimes.defaults[LIVE_HELPER_RUNTIME].exec(["builder/bash/run_profile_config.sh", str(self.env_file(self.context("config", "profile", profile_name=profile_name), profile_artifacts_dir(self.root, profile_name) / "profile-config.env"))]) + apply_profile_common_configuration(self.root, self.root / "live", profile_name, profile) for base_name, _ in base_chain: - inject_module_resources(base_dir(self.root, base_name), live_dir, base_name) + inject_module_resources(base_dir(self.root, base_name), self.root / "live", base_name) def inject_features(self, profile_name: str, base_name: str, feature_names: Iterable[str]) -> None: - pre_feature_ctx = ctx(self.root, "pre-feature", "base", profile_name, base_name=base_name) - pre_feature_env = write_phase_env(pre_feature_ctx, base_artifacts_dir(self.root, base_name) / "pre-feature.env", self.fake_legacy) - self.python_phase("base", base_name, "pre-feature", profile=profile_name, base=base_name) - self.shell_phase("base", base_name, "pre-feature", pre_feature_env) + pre = self.context("pre-feature", "base", profile_name=profile_name, base_name=base_name) + pre_env = self.env_file(pre, base_artifacts_dir(self.root, base_name) / "pre-feature.env") + self.run_python("base", base_name, "pre-feature", profile_name=profile_name, base_name=base_name) + self.run_shell("base", base_name, "pre-feature", pre_env) for feature_name in feature_names: - feature_spec = load_feature_spec(self.root, feature_name) - save_feature_metadata(profile_artifacts_dir(self.root, profile_name) / "features" / f"{feature_name}.json", feature_name, feature_spec) - pre_ctx = ctx(self.root, "pre-inj", "feature", profile_name, feature_name=feature_name) - pre_env = write_phase_env(pre_ctx, feature_artifacts_dir(self.root, feature_name) / "pre-inj.env", self.fake_legacy) - self.python_phase("feature", feature_name, "pre-inj", profile=profile_name, feature=feature_name) - self.shell_phase("feature", feature_name, "pre-inj", pre_env) + save_feature_metadata(profile_artifacts_dir(self.root, profile_name) / "features" / f"{feature_name}.json", feature_name, load_module_spec(self.root, "feature", feature_name)) + pre = self.context("pre-inj", "feature", profile_name=profile_name, base_name=base_name, feature_name=feature_name) + pre_env = self.env_file(pre, feature_artifacts_dir(self.root, feature_name) / "pre-inj.env") + self.run_python("feature", feature_name, "pre-inj", profile_name=profile_name, base_name=base_name, feature_name=feature_name) + self.run_shell("feature", feature_name, "pre-inj", pre_env) inject_module_resources(feature_dir(self.root, feature_name), self.root / "live", feature_name) - post_ctx = ctx(self.root, "post-inj", "feature", profile_name, feature_name=feature_name) - post_env = write_phase_env(post_ctx, feature_artifacts_dir(self.root, feature_name) / "post-inj.env", self.fake_legacy) - self.shell_phase("feature", feature_name, "post-inj", post_env) - self.python_phase("feature", feature_name, "post-inj", profile=profile_name, feature=feature_name) - post_feature_ctx = ctx(self.root, "post-feature", "base", profile_name, base_name=base_name) - post_feature_env = write_phase_env(post_feature_ctx, base_artifacts_dir(self.root, base_name) / "post-feature.env", self.fake_legacy) - self.python_phase("base", base_name, "post-feature", profile=profile_name, base=base_name) - self.shell_phase("base", base_name, "post-feature", post_feature_env) + post = self.context("post-inj", "feature", profile_name=profile_name, base_name=base_name, feature_name=feature_name) + post_env = self.env_file(post, feature_artifacts_dir(self.root, feature_name) / "post-inj.env") + self.run_shell("feature", feature_name, "post-inj", post_env) + self.run_python("feature", feature_name, "post-inj", profile_name=profile_name, base_name=base_name, feature_name=feature_name) + post = self.context("post-feature", "base", profile_name=profile_name, base_name=base_name) + post_env = self.env_file(post, base_artifacts_dir(self.root, base_name) / "post-feature.env") + self.run_python("base", base_name, "post-feature", profile_name=profile_name, base_name=base_name) + self.run_shell("base", base_name, "post-feature", post_env) def finalize_profile(self, profile_name: str) -> None: profile_pre_build(self.root / "live", profile_name) - build_ctx = ctx(self.root, "build", "profile", profile_name) - self.run_profile_build(profile_name, write_phase_env(build_ctx, profile_artifacts_dir(self.root, profile_name) / "profile-build.env", self.fake_legacy)) + build_env = self.env_file(self.context("build", "profile", profile_name=profile_name), profile_artifacts_dir(self.root, profile_name) / "profile-build.env") + self.runtimes.defaults[LIVE_HELPER_RUNTIME].exec(["builder/bash/run_profile_build.sh", str(build_env)]) profile_finalize(self.root, profile_name) def build_profile(self, profile_name: str) -> None: @@ -315,14 +279,12 @@ class Orchestrator: self.inject_features(profile_name, profile.base, profile.features) self.finalize_profile(profile_name) - def run(self, profile_name: str | None, all_profiles: bool) -> None: - self.runtimes.start_defaults() + def run(self, all_profiles: bool, profile_name: str = "") -> None: + self.runtimes.start() try: self.validate() - hint = profile_name or "demo" - self.build_common_features(hint) - names = list_names(self.root / "profiles") if all_profiles else [profile_name] - for name in names: + self.build_common_features(profile_name or "demo") + for name in (list_names(self.root / "profiles") if all_profiles else [profile_name]): self.build_profile(name) finally: self.runtimes.close() @@ -337,28 +299,29 @@ def parser() -> argparse.ArgumentParser: grp.add_argument("--profile") grp.add_argument("--all-profiles", action="store_true") for runtime in (ORCHESTRATOR_RUNTIME, PACKAGE_BUILDER_RUNTIME, LIVE_HELPER_RUNTIME): - arg = runtime.replace("-", "_") - runp.add_argument(f"--{arg}-image", default="") - runp.add_argument(f"--{arg}-dockerfile", default=DEFAULT_RUNTIME_DOCKERFILES[runtime]) - runp.add_argument(f"--{arg}-context", default=".") - runp.add_argument("--keep-containers", action="store_true") - runp.add_argument("--fake-legacy", action="store_true") + opt = runtime.replace("-", "_") + runp.add_argument(f"--{opt}-image", default="") + runp.add_argument(f"--{opt}-dockerfile", default=DEFAULT_RUNTIME_DOCKERFILES[runtime]) + runp.add_argument(f"--{opt}-context", default=".") return p def main() -> int: args = parser().parse_args() root = root_dir() + if args.command == "validate": + Orchestrator(root, RuntimePool(root, { + ORCHESTRATOR_RUNTIME: RuntimeConfig(image="noop"), + PACKAGE_BUILDER_RUNTIME: RuntimeConfig(image="noop"), + LIVE_HELPER_RUNTIME: RuntimeConfig(image="noop"), + }, keep=True)).validate() + return 0 runtimes = RuntimePool(root, { ORCHESTRATOR_RUNTIME: RuntimeConfig(args.orchestrator_image, args.orchestrator_dockerfile, args.orchestrator_context), PACKAGE_BUILDER_RUNTIME: RuntimeConfig(args.package_builder_image, args.package_builder_dockerfile, args.package_builder_context), LIVE_HELPER_RUNTIME: RuntimeConfig(args.live_helper_image, args.live_helper_dockerfile, args.live_helper_context), }, keep=args.keep_containers) - orch = Orchestrator(root, runtimes, fake_legacy=args.fake_legacy) - if args.command == "validate": - orch.validate() - else: - orch.run(args.profile, args.all_profiles) + Orchestrator(root, runtimes).run(args.all_profiles, args.profile or "") return 0 diff --git a/builder/py/retrobuilder/context.py b/builder/py/retrobuilder/context.py index 1b1c98a..21e84c7 100644 --- a/builder/py/retrobuilder/context.py +++ b/builder/py/retrobuilder/context.py @@ -1,8 +1,24 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import Mapping +from typing import Any, Mapping + + +def _norm(name: str) -> str: + return "".join(ch if ch.isalnum() else "_" for ch in name.upper()) + + +class Node(dict): + __getattr__ = dict.get + + @classmethod + def wrap(cls, value: Any) -> Any: + if isinstance(value, dict): + return cls({k: cls.wrap(v) for k, v in value.items()}) + if isinstance(value, (list, tuple)): + return [cls.wrap(v) for v in value] + return value @dataclass(frozen=True) @@ -11,59 +27,91 @@ class BuildContext: live_dir: Path artifacts_root: Path phase: str - current_kind: str - current_name: str - current_module_artifact_dir: Path - profile_name: str = '' - profile_artifact_dir: Path | None = None - base_name: str = '' - base_artifact_dir: Path | None = None - feature_name: str = '' - feature_artifact_dir: Path | None = None - profile_features: tuple[str, ...] = () + kind: str + name: str + profile_name: str = "" + base_name: str = "" + feature_name: str = "" + profile: Mapping[str, Any] = field(default_factory=dict) + base: Mapping[str, Any] = field(default_factory=dict) + features: Mapping[str, Mapping[str, Any]] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "profile", Node.wrap(dict(self.profile))) + object.__setattr__(self, "base", Node.wrap(dict(self.base))) + object.__setattr__(self, "features", {k: Node.wrap(dict(v)) for k, v in self.features.items()}) def to_env(self) -> dict[str, str]: env = { - 'PROJECT_ROOT': str(self.project_root), - 'LIVE_DIR': str(self.live_dir), - 'ARTIFACTS_ROOT': str(self.artifacts_root), - 'PHASE': self.phase, - 'CURRENT_KIND': self.current_kind, - 'CURRENT_NAME': self.current_name, - 'CURRENT_MODULE_ARTIFACT_DIR': str(self.current_module_artifact_dir), - 'PROFILE_NAME': self.profile_name, - 'BASE_NAME': self.base_name, - 'FEATURE_NAME': self.feature_name, - 'PROFILE_FEATURES': '|'.join(self.profile_features), + "PROJECT_ROOT": str(self.project_root), + "LIVE_DIR": str(self.live_dir), + "ARTIFACTS_ROOT": str(self.artifacts_root), + "PHASE": self.phase, + "CURRENT_KIND": self.kind, + "CURRENT_NAME": self.name, + "PROFILE_NAME": self.profile_name, + "BASE_NAME": self.base_name, + "FEATURE_NAME": self.feature_name, + "FEATURE_NAMES": "|".join(self.features.keys()), } - if self.profile_artifact_dir is not None: - env['PROFILE_ARTIFACT_DIR'] = str(self.profile_artifact_dir) - if self.base_artifact_dir is not None: - env['BASE_ARTIFACT_DIR'] = str(self.base_artifact_dir) - if self.feature_artifact_dir is not None: - env['FEATURE_ARTIFACT_DIR'] = str(self.feature_artifact_dir) + env.update(_flatten("PROFILE", self.profile)) + env.update(_flatten("BASE", self.base)) + for feat_name, feat_cfg in self.features.items(): + env.update(_flatten(f"FEAT__{_norm(feat_name)}", feat_cfg)) return env @classmethod - def from_env(cls, env: Mapping[str, str]) -> 'BuildContext': - def p(name: str) -> Path | None: - value = env.get(name, '') - return Path(value) if value else None - - features = tuple(filter(None, env.get('PROFILE_FEATURES', '').split('|'))) + def from_env(cls, env: Mapping[str, str]) -> "BuildContext": + feature_names = [x for x in env.get("FEATURE_NAMES", "").split("|") if x] + profile = _inflate("PROFILE", env) + base = _inflate("BASE", env) + features = {name: _inflate(f"FEAT__{_norm(name)}", env) for name in feature_names} return cls( - project_root=Path(env['PROJECT_ROOT']), - live_dir=Path(env['LIVE_DIR']), - artifacts_root=Path(env['ARTIFACTS_ROOT']), - phase=env['PHASE'], - current_kind=env['CURRENT_KIND'], - current_name=env['CURRENT_NAME'], - current_module_artifact_dir=Path(env['CURRENT_MODULE_ARTIFACT_DIR']), - profile_name=env.get('PROFILE_NAME', ''), - profile_artifact_dir=p('PROFILE_ARTIFACT_DIR'), - base_name=env.get('BASE_NAME', ''), - base_artifact_dir=p('BASE_ARTIFACT_DIR'), - feature_name=env.get('FEATURE_NAME', ''), - feature_artifact_dir=p('FEATURE_ARTIFACT_DIR'), - profile_features=features, + project_root=Path(env["PROJECT_ROOT"]), + live_dir=Path(env["LIVE_DIR"]), + artifacts_root=Path(env["ARTIFACTS_ROOT"]), + phase=env["PHASE"], + kind=env["CURRENT_KIND"], + name=env["CURRENT_NAME"], + profile_name=env.get("PROFILE_NAME", ""), + base_name=env.get("BASE_NAME", ""), + feature_name=env.get("FEATURE_NAME", ""), + profile=profile, + base=base, + features=features, ) + + +def _stringify(value: Any) -> str: + if isinstance(value, bool): + return "1" if value else "0" + if value is None: + return "" + if isinstance(value, (list, tuple)): + return "|".join(_stringify(v) for v in value) + return str(value) + + +def _flatten(prefix: str, data: Mapping[str, Any]) -> dict[str, str]: + out: dict[str, str] = {} + for key, value in data.items(): + name = f"{prefix}__{_norm(str(key))}" + if isinstance(value, dict): + out.update(_flatten(name, value)) + else: + out[name] = _stringify(value) + return out + + +def _inflate(prefix: str, env: Mapping[str, str]) -> dict[str, Any]: + root: dict[str, Any] = {} + marker = f"{prefix}__" + for key, value in env.items(): + if not key.startswith(marker): + continue + parts = key[len(marker):].split("__") + node = root + for part in parts[:-1]: + node = node.setdefault(part.lower(), {}) + node[parts[-1].lower()] = value + return root diff --git a/builder/py/retrobuilder/entrypoints.py b/builder/py/retrobuilder/entrypoints.py index 7664f5e..62d8015 100644 --- a/builder/py/retrobuilder/entrypoints.py +++ b/builder/py/retrobuilder/entrypoints.py @@ -1,57 +1,38 @@ from __future__ import annotations -import json +import os import sys from abc import ABC -from pathlib import Path from retrobuilder.context import BuildContext class BaseEntry(ABC): - def pre_gen(self, ctx: BuildContext) -> None: - return None - - def post_gen(self, ctx: BuildContext) -> None: - return None - - def pre_feature(self, ctx: BuildContext) -> None: - return None - - def post_feature(self, ctx: BuildContext) -> None: - return None + def pre_gen(self, ctx: BuildContext) -> None: return None + def post_gen(self, ctx: BuildContext) -> None: return None + def pre_feature(self, ctx: BuildContext) -> None: return None + def post_feature(self, ctx: BuildContext) -> None: return None class FeatureEntry(ABC): - def pre_gen(self, ctx: BuildContext) -> None: - return None - - def post_gen(self, ctx: BuildContext) -> None: - return None - - def pre_inj(self, ctx: BuildContext) -> None: - return None - - def post_inj(self, ctx: BuildContext) -> None: - return None + def pre_gen(self, ctx: BuildContext) -> None: return None + def post_gen(self, ctx: BuildContext) -> None: return None + def pre_inj(self, ctx: BuildContext) -> None: return None + def post_inj(self, ctx: BuildContext) -> None: return None def cli_dispatch(spec, entry_cls) -> int: if len(sys.argv) < 2: - print('Usage: entry.py ', file=sys.stderr) + print("Usage: entry.py ", file=sys.stderr) return 2 - command = sys.argv[1] - if command == 'spec': - print(json.dumps(spec.to_dict(), indent=2, sort_keys=True)) + if command == "spec": + print(spec.to_dict()) return 0 - - ctx = BuildContext.from_env(dict(__import__('os').environ)) entry = entry_cls() - method_name = command.replace('-', '_') - method = getattr(entry, method_name, None) + method = getattr(entry, command.replace("-", "_"), None) if method is None: - print(f'Unsupported phase: {command}', file=sys.stderr) + print(f"Unsupported phase: {command}", file=sys.stderr) return 2 - method(ctx) + method(BuildContext.from_env(os.environ)) return 0 diff --git a/builder/py/retrobuilder/loader.py b/builder/py/retrobuilder/loader.py index 388d8a0..d8ab8f4 100644 --- a/builder/py/retrobuilder/loader.py +++ b/builder/py/retrobuilder/loader.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util from pathlib import Path from types import ModuleType +from typing import Any from retrobuilder.model import BaseSpec, FeatureSpec, ProfileSpec from retrobuilder.paths import base_dir, feature_dir, profile_file @@ -11,71 +12,71 @@ from retrobuilder.paths import base_dir, feature_dir, profile_file def _load_module(path: Path, name: str) -> ModuleType: spec = importlib.util.spec_from_file_location(name, path) if spec is None or spec.loader is None: - raise RuntimeError(f'Cannot load module from {path}') + raise RuntimeError(f"Cannot load module from {path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def load_profile(root: Path, name: str) -> ProfileSpec: - path = profile_file(root, name) - module = _load_module(path, f'profile_{name}') - profile = getattr(module, 'PROFILE', None) + module = _load_module(profile_file(root, name), f"profile_{name}") + profile = getattr(module, "PROFILE", None) if not isinstance(profile, ProfileSpec): - raise TypeError(f'{path} must define PROFILE as ProfileSpec') + raise TypeError(f"profiles/{name}.py must define PROFILE as ProfileSpec") return profile -def load_base_module(root: Path, name: str) -> ModuleType: - return _load_module(base_dir(root, name) / 'entry.py', f'base_{name}') +def _load_module_py(root: Path, kind: str, name: str) -> ModuleType: + path = (base_dir if kind == "base" else feature_dir)(root, name) / "entry.py" + return _load_module(path, f"{kind}_{name}") -def load_feature_module(root: Path, name: str) -> ModuleType: - return _load_module(feature_dir(root, name) / 'entry.py', f'feature_{name}') - - -def load_base_spec(root: Path, name: str) -> BaseSpec: - module = load_base_module(root, name) - spec = getattr(module, 'SPEC', None) - if not isinstance(spec, BaseSpec): - raise TypeError(f'bases/{name}/entry.py must define SPEC as BaseSpec') +def load_module_spec(root: Path, kind: str, name: str): + module = _load_module_py(root, kind, name) + spec = getattr(module, "SPEC", None) + expected = BaseSpec if kind == "base" else FeatureSpec + if not isinstance(spec, expected): + raise TypeError(f"{kind}s/{name}/entry.py must define SPEC as {expected.__name__}") return spec -def load_base_chain(root: Path, name: str) -> list[tuple[str, BaseSpec]]: - chain: list[tuple[str, BaseSpec]] = [] +def load_module_entry(root: Path, kind: str, name: str): + module = _load_module_py(root, kind, name) + entry = getattr(module, "Entry", None) + if entry is None: + raise TypeError(f"{kind}s/{name}/entry.py must define Entry") + return entry + + +def load_base_chain(root: Path, name: str): + return load_module_chain(root, "base", name) + + +def load_module_chain(root: Path, kind: str, name: str): + chain: list[tuple[str, Any]] = [] seen: set[str] = set() current = name while current: if current in seen: - raise RuntimeError(f'Base inheritance loop detected at {current}') + raise RuntimeError(f"{kind} inheritance loop detected at {current}") seen.add(current) - spec = load_base_spec(root, current) + spec = load_module_spec(root, kind, current) chain.append((current, spec)) - current = spec.parent or '' + current = spec.parent or "" chain.reverse() return chain -def load_feature_spec(root: Path, name: str) -> FeatureSpec: - module = load_feature_module(root, name) - spec = getattr(module, 'SPEC', None) - if not isinstance(spec, FeatureSpec): - raise TypeError(f'features/{name}/entry.py must define SPEC as FeatureSpec') - return spec +def resolve_module_config(root: Path, kind: str, name: str) -> dict[str, Any]: + merged: dict[str, Any] = {} + for _name, spec in load_module_chain(root, kind, name): + merged.update(spec.config) + return merged -def load_base_entry(root: Path, name: str): - module = load_base_module(root, name) - entry_cls = getattr(module, 'Entry', None) - if entry_cls is None: - raise TypeError(f'bases/{name}/entry.py must define Entry') - return entry_cls - - -def load_feature_entry(root: Path, name: str): - module = load_feature_module(root, name) - entry_cls = getattr(module, 'Entry', None) - if entry_cls is None: - raise TypeError(f'features/{name}/entry.py must define Entry') - return entry_cls +def resolve_module_docker_override(root: Path, kind: str, name: str, phase: str): + resolved = None + for _name, spec in load_module_chain(root, kind, name): + if phase in spec.docker_overrides: + resolved = spec.docker_overrides[phase] + return resolved diff --git a/builder/py/retrobuilder/model.py b/builder/py/retrobuilder/model.py index acdbec6..ffd20c5 100644 --- a/builder/py/retrobuilder/model.py +++ b/builder/py/retrobuilder/model.py @@ -1,69 +1,34 @@ from __future__ import annotations -from dataclasses import dataclass, field, asdict -from typing import Any +from dataclasses import asdict, dataclass, field +from typing import Any, Mapping @dataclass(frozen=True) class DockerStageSpec: - image: str | None = None - dockerfile: str | None = None - docker_context: str | None = None + image: str = "" + dockerfile: str = "" + docker_context: str = "." - def __post_init__(self) -> None: - if self.image is not None and not isinstance(self.image, str): - raise TypeError('DockerStageSpec.image must be str or None') - if self.dockerfile is not None and not isinstance(self.dockerfile, str): - raise TypeError('DockerStageSpec.dockerfile must be str or None') - if self.docker_context is not None and not isinstance(self.docker_context, str): - raise TypeError('DockerStageSpec.docker_context must be str or None') + def is_set(self) -> bool: + return bool(self.image or self.dockerfile) - @classmethod - def from_value(cls, value: Any) -> 'DockerStageSpec': - if isinstance(value, DockerStageSpec): - return value - if isinstance(value, dict): - return cls( - image=value.get('image'), - dockerfile=value.get('dockerfile'), - docker_context=value.get('docker_context'), - ) - raise TypeError(f'Expected DockerStageSpec or dict, got {type(value)!r}') + +@dataclass(frozen=True) +class ModuleSpec: + description: str = "" + parent: str = "" + docker_overrides: Mapping[str, DockerStageSpec] = field(default_factory=dict) + config: Mapping[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: return asdict(self) -def _docker_overrides(values: Any) -> dict[str, DockerStageSpec]: - if values is None: - return {} - if not isinstance(values, dict): - raise TypeError(f'Expected dict/None for docker_overrides, got {type(values)!r}') - result: dict[str, DockerStageSpec] = {} - for phase, item in values.items(): - if not isinstance(phase, str): - raise TypeError('docker_overrides keys must be strings') - result[phase] = DockerStageSpec.from_value(item) - return result - - -@dataclass(frozen=True) -class ModuleSpec: - description: str = '' - docker_overrides: dict[str, DockerStageSpec] = field(default_factory=dict) - - def __post_init__(self) -> None: - object.__setattr__(self, 'docker_overrides', _docker_overrides(self.docker_overrides)) - - def to_dict(self) -> dict[str, Any]: - payload = asdict(self) - payload['docker_overrides'] = {name: spec.to_dict() for name, spec in self.docker_overrides.items()} - return payload - - @dataclass(frozen=True) class BaseSpec(ModuleSpec): - parent: str | None = None + BASE__REPO_URL: str = "http://archive.debian.org/debian/" + BASE__SECURITY_REPO_URL: str = "http://archive.debian.org/debian-security/" @dataclass(frozen=True) @@ -74,16 +39,15 @@ class FeatureSpec(ModuleSpec): @dataclass(frozen=True) class ProfileSpec: base: str - features: tuple[str, ...] = field(default_factory=tuple) - edition: str = '' - description: str = '' - splash: str | None = None + features: tuple[str, ...] = () + edition: str = "" + description: str = "" + splash: str = "" + config: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - if isinstance(self.features, list): - object.__setattr__(self, 'features', tuple(str(v) for v in self.features)) - elif not isinstance(self.features, tuple): - raise TypeError('ProfileSpec.features must be list or tuple') + if not isinstance(self.features, tuple): + object.__setattr__(self, "features", tuple(self.features)) def to_dict(self) -> dict[str, Any]: return asdict(self)