opt

Jenkinsfile

@@ -15,210 +15,236 @@ pipeline {
     }
 
     stages {
-        stage('Init runtime images and validate builder') {
+        stage('Build runtimes') {
             steps {
                 script {
-                    def json = new groovy.json.JsonSlurperClassic()
+                    env.RUNTIME_BUILDER_IMAGE = docker.build(
-                    def runtimeImages = [:]
+                        "retrodebian/builder-alpine:${env.BUILD_TAG}",
-                    def moduleDockerCache = [:]
+                        "-f ${env.PYTHON_DOCKERFILE} ."
-                    def profileCache = [:]
+                    ).imageName()
 
-                    def buildRuntimeImage = { String runtimeName, String dockerfilePath ->
+                    env.RUNTIME_ETCH_IMAGE = docker.build(
-                        return docker.build(
+                        "retrodebian/package-builder-etch:${env.BUILD_TAG}",
-                            "retrodebian/${runtimeName}:${env.BUILD_TAG}",
+                        "-f ${env.ETCH_DOCKERFILE} ."
-                            "-f ${dockerfilePath} ."
+                    ).imageName()
-                        ).imageName()
+
+                    env.RUNTIME_LENNY_IMAGE = docker.build(
+                        "retrodebian/live-helper:${env.BUILD_TAG}",
+                        "-f ${env.LENNY_DOCKERFILE} ."
+                    ).imageName()
+                }
+            }
+        }
+
+        stage('Validate builder') {
+            steps {
+                script {
+                    docker.image(env.RUNTIME_BUILDER_IMAGE).inside {
+                        sh 'python3 builder/py/build.py validate'
                     }
+                }
+            }
+        }
 
-                    runtimeImages.python = buildRuntimeImage('builder-alpine', env.PYTHON_DOCKERFILE)
+        stage('Build common features and profiles') {
-                    runtimeImages.etch   = buildRuntimeImage('package-builder-etch', env.ETCH_DOCKERFILE)
+            steps {
-                    runtimeImages.lenny  = buildRuntimeImage('live-helper', env.LENNY_DOCKERFILE)
+                script {
+                    def jsonSlurper = new groovy.json.JsonSlurperClassic()
+                    def dockerOverrideCache = [:]
+                    def profileResolveCache = [:]
 
-                    def withPython = { Closure body ->
+                    def withImage = { String imageName, Closure body ->
-                        docker.image(runtimeImages.python).inside {
+                        docker.image(imageName).inside {
                             body()
                         }
                     }
 
-                    def shJson = { String scriptText ->
+                    def withBuilder = { Closure body ->
-                        def raw = sh(script: scriptText, returnStdout: true).trim()
+                        withImage(env.RUNTIME_BUILDER_IMAGE, body)
-                        return raw ? json.parseText(raw) : [:]
                     }
 
-                    def resolveProfile = { String profileName ->
+                    def builderJson = { String command ->
-                        if (!profileCache.containsKey(profileName)) {
+                        def raw = ''
-                            withPython {
+                        withBuilder {
-                                profileCache[profileName] = shJson("python3 builder/py/build.py profile-resolve --profile ${profileName}")
+                            raw = sh(script: command, returnStdout: true).trim()
-                            }
                         }
-                        return profileCache[profileName]
+                        return raw ? jsonSlurper.parseText(raw) : [:]
-                    }
-
-                    def resolveModuleDocker = { String kind, String name, String phase ->
-                        def cacheKey = "${kind}:${name}:${phase}"
-                        if (!moduleDockerCache.containsKey(cacheKey)) {
-                            withPython {
-                                moduleDockerCache[cacheKey] = shJson(
-                                    "python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase}"
-                                )
-                            }
-                        }
-                        return moduleDockerCache[cacheKey]
                     }
 
                     def resolveModuleRuntimeImage = { String runtimeName, String kind, String name, String phase, String defaultImage ->
-                        def overrideCfg = resolveModuleDocker(kind, name, phase)
+                        def cacheKey = "${runtimeName}:${kind}:${name}:${phase}"
-                        def overrideImage = (overrideCfg.image ?: '').trim()
+                        if (dockerOverrideCache.containsKey(cacheKey)) {
-                        def overrideDockerfile = (overrideCfg.dockerfile ?: '').trim()
+                            return dockerOverrideCache[cacheKey]
-                        def overrideContext = (overrideCfg.docker_context ?: '.').trim()
+                        }
 
+                        def payload = builderJson("python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase} --json")
+                        def overrideImage = (payload.image ?: '').toString().trim()
+                        def overrideDockerfile = (payload.dockerfile ?: '').toString().trim()
+                        def overrideContext = (payload.docker_context ?: '').toString().trim()
+
+                        def resolved = defaultImage
                         if (overrideDockerfile) {
                             def tag = overrideImage ? overrideImage : "retrodebian/${runtimeName}-${kind}-${name}-${phase}:${env.BUILD_TAG}"
-                            return docker.build(tag, "-f ${overrideDockerfile} ${overrideContext}").imageName()
+                            def contextPath = overrideContext ? overrideContext : '.'
+                            resolved = docker.build(tag, "-f ${overrideDockerfile} ${contextPath}").imageName()
+                        } else if (overrideImage) {
+                            resolved = overrideImage
                         }
-                        if (overrideImage) {
+
-                            return overrideImage
+                        dockerOverrideCache[cacheKey] = resolved
-                        }
+                        return resolved
-                        return defaultImage
                     }
 
-                    withPython {
+                    def resolveProfile = { String profileName ->
-                        sh 'python3 builder/py/build.py validate'
+                        if (profileResolveCache.containsKey(profileName)) {
+                            return profileResolveCache[profileName]
+                        }
+                        def payload = builderJson("python3 builder/py/build.py profile-resolve --profile ${profileName}")
+                        profileResolveCache[profileName] = payload
+                        return payload
                     }
 
                     def featureNames = []
-                    withPython {
+                    withBuilder {
                         def raw = sh(script: 'python3 builder/py/build.py list features', returnStdout: true).trim()
                         featureNames = raw ? raw.split('\n').findAll { it?.trim() } : []
                     }
 
                     for (featureName in featureNames) {
                         stage("Feature ${featureName}") {
-                            def pythonPreImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-gen', runtimeImages.python)
+                            def preGenPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-gen', env.RUNTIME_BUILDER_IMAGE)
-                            docker.image(pythonPreImage).inside {
+                            def generateEtchImage = resolveModuleRuntimeImage('etch', 'feature', featureName, 'generate', env.RUNTIME_ETCH_IMAGE)
+                            def postGenPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-gen', env.RUNTIME_BUILDER_IMAGE)
+
+                            withImage(preGenPythonImage) {
                                 sh "python3 builder/py/build.py run-python-phase --kind feature --phase pre-gen --profile ${params.PROFILE} --feature ${featureName}"
                                 sh "python3 builder/py/build.py export-env --kind feature --phase pre-gen --profile ${params.PROFILE} --feature ${featureName} --output artifacts/features/${featureName}/runtime.env"
                             }
 
-                            def etchImage = resolveModuleRuntimeImage('etch', 'feature', featureName, 'generate', runtimeImages.etch)
+                            withImage(generateEtchImage) {
-                            docker.image(etchImage).inside {
                                 sh "builder/bash/run_generate.sh features/${featureName} artifacts/features/${featureName}/runtime.env"
                             }
 
-                            def pythonPostImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-gen', runtimeImages.python)
+                            withImage(postGenPythonImage) {
-                            docker.image(pythonPostImage).inside {
                                 sh "python3 builder/py/build.py run-python-phase --kind feature --phase post-gen --profile ${params.PROFILE} --feature ${featureName}"
                             }
                         }
                     }
 
                     def profileNames = []
-                    if (params.BUILD_ALL_PROFILES) {
+                    withBuilder {
-                        withPython {
+                        if (params.BUILD_ALL_PROFILES) {
                             def raw = sh(script: 'python3 builder/py/build.py list profiles', returnStdout: true).trim()
                             profileNames = raw ? raw.split('\n').findAll { it?.trim() } : []
+                        } else {
+                            profileNames = [params.PROFILE]
                         }
-                    } else {
-                        profileNames = [params.PROFILE]
                     }
 
                     for (profileName in profileNames) {
                         stage("Profile ${profileName}") {
-                            def profileInfo = resolveProfile(profileName)
+                            def profileData = resolveProfile(profileName)
-                            def baseName = profileInfo.base
+                            def baseName = profileData.base.toString()
-                            def baseChain = profileInfo.base_chain ?: []
+                            def baseChain = (profileData.base_chain ?: []) as List
-                            def featureList = profileInfo.features ?: []
+                            def featureList = (profileData.features ?: []) as List
 
                             for (baseItem in baseChain) {
                                 stage("Base ${profileName} / ${baseItem}") {
-                                    def pythonPreImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'pre-gen', runtimeImages.python)
+                                    def preGenPythonImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'pre-gen', env.RUNTIME_BUILDER_IMAGE)
-                                    docker.image(pythonPreImage).inside {
+                                    def generateEtchImage = resolveModuleRuntimeImage('etch', 'base', baseItem, 'generate', env.RUNTIME_ETCH_IMAGE)
+                                    def postGenPythonImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'post-gen', env.RUNTIME_BUILDER_IMAGE)
+
+                                    withImage(preGenPythonImage) {
                                         sh "python3 builder/py/build.py run-python-phase --kind base --phase pre-gen --profile ${profileName} --base ${baseItem}"
                                         sh "python3 builder/py/build.py export-env --kind base --phase pre-gen --profile ${profileName} --base ${baseItem} --output artifacts/bases/${baseItem}/runtime.env"
                                     }
 
-                                    def etchImage = resolveModuleRuntimeImage('etch', 'base', baseItem, 'generate', runtimeImages.etch)
+                                    withImage(generateEtchImage) {
-                                    docker.image(etchImage).inside {
                                         sh "builder/bash/run_generate.sh bases/${baseItem} artifacts/bases/${baseItem}/runtime.env"
                                     }
 
-                                    def pythonPostImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'post-gen', runtimeImages.python)
+                                    withImage(postGenPythonImage) {
-                                    docker.image(pythonPostImage).inside {
                                         sh "python3 builder/py/build.py run-python-phase --kind base --phase post-gen --profile ${profileName} --base ${baseItem}"
                                     }
                                 }
                             }
 
-                            withPython {
+                            withBuilder {
                                 sh "python3 builder/py/build.py prepare-profile --profile ${profileName}"
                                 sh "python3 builder/py/build.py export-env --kind profile --phase config --profile ${profileName} --output artifacts/profiles/${profileName}/profile-config.env"
                                 sh "python3 builder/py/build.py inject-resources --kind base --name ${baseName}"
                             }
 
-                            docker.image(runtimeImages.lenny).inside {
+                            withImage(env.RUNTIME_LENNY_IMAGE) {
                                 sh "builder/bash/run_profile_config.sh artifacts/profiles/${profileName}/profile-config.env"
                             }
 
-                            def basePreFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'pre-feature', runtimeImages.python)
+                            def basePreFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'pre-feature', env.RUNTIME_BUILDER_IMAGE)
-                            docker.image(basePreFeaturePythonImage).inside {
+                            def basePreFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'pre-feature', env.RUNTIME_LENNY_IMAGE)
+
+                            withImage(basePreFeaturePythonImage) {
                                 sh "python3 builder/py/build.py run-python-phase --kind base --phase pre-feature --profile ${profileName} --base ${baseName}"
                                 sh "python3 builder/py/build.py export-env --kind base --phase pre-feature --profile ${profileName} --base ${baseName} --output artifacts/bases/${baseName}/pre-feature.env"
                             }
 
-                            def basePreFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'pre-feature', runtimeImages.lenny)
+                            withImage(basePreFeatureLennyImage) {
-                            docker.image(basePreFeatureLennyImage).inside {
                                 sh "builder/bash/run_entry.sh bases/${baseName} artifacts/bases/${baseName}/pre-feature.env pre-feature"
                             }
 
                             for (featureName in featureList) {
                                 stage("Inject ${profileName} / ${featureName}") {
-                                    def featurePreInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-inj', runtimeImages.python)
+                                    def featurePreInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-inj', env.RUNTIME_BUILDER_IMAGE)
-                                    docker.image(featurePreInjPythonImage).inside {
+                                    def featurePreInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'pre-inj', env.RUNTIME_LENNY_IMAGE)
+                                    def featurePostInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'post-inj', env.RUNTIME_LENNY_IMAGE)
+                                    def featurePostInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-inj', env.RUNTIME_BUILDER_IMAGE)
+
+                                    withBuilder {
                                         sh "python3 builder/py/build.py save-feature-metadata --profile ${profileName} --feature ${featureName}"
+                                    }
+
+                                    withImage(featurePreInjPythonImage) {
                                         sh "python3 builder/py/build.py run-python-phase --kind feature --phase pre-inj --profile ${profileName} --feature ${featureName}"
                                         sh "python3 builder/py/build.py export-env --kind feature --phase pre-inj --profile ${profileName} --feature ${featureName} --output artifacts/features/${featureName}/pre-inj.env"
                                     }
 
-                                    def featurePreInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'pre-inj', runtimeImages.lenny)
+                                    withImage(featurePreInjLennyImage) {
-                                    docker.image(featurePreInjLennyImage).inside {
                                         sh "builder/bash/run_entry.sh features/${featureName} artifacts/features/${featureName}/pre-inj.env pre-inj"
                                     }
 
-                                    def featurePostInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-inj', runtimeImages.python)
+                                    withBuilder {
-                                    docker.image(featurePostInjPythonImage).inside {
                                         sh "python3 builder/py/build.py inject-resources --kind feature --name ${featureName}"
                                         sh "python3 builder/py/build.py export-env --kind feature --phase post-inj --profile ${profileName} --feature ${featureName} --output artifacts/features/${featureName}/post-inj.env"
                                     }
 
-                                    def featurePostInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'post-inj', runtimeImages.lenny)
+                                    withImage(featurePostInjLennyImage) {
-                                    docker.image(featurePostInjLennyImage).inside {
                                         sh "builder/bash/run_entry.sh features/${featureName} artifacts/features/${featureName}/post-inj.env post-inj"
                                     }
 
-                                    docker.image(featurePostInjPythonImage).inside {
+                                    withImage(featurePostInjPythonImage) {
                                         sh "python3 builder/py/build.py run-python-phase --kind feature --phase post-inj --profile ${profileName} --feature ${featureName}"
                                     }
                                 }
                             }
 
-                            def basePostFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'post-feature', runtimeImages.python)
+                            def basePostFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'post-feature', env.RUNTIME_BUILDER_IMAGE)
-                            docker.image(basePostFeaturePythonImage).inside {
+                            def basePostFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'post-feature', env.RUNTIME_LENNY_IMAGE)
+
+                            withImage(basePostFeaturePythonImage) {
                                 sh "python3 builder/py/build.py run-python-phase --kind base --phase post-feature --profile ${profileName} --base ${baseName}"
                                 sh "python3 builder/py/build.py export-env --kind base --phase post-feature --profile ${profileName} --base ${baseName} --output artifacts/bases/${baseName}/post-feature.env"
                                 sh "python3 builder/py/build.py profile-pre-build --profile ${profileName}"
                                 sh "python3 builder/py/build.py export-env --kind profile --phase build --profile ${profileName} --output artifacts/profiles/${profileName}/profile-build.env"
                             }
 
-                            def basePostFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'post-feature', runtimeImages.lenny)
+                            withImage(basePostFeatureLennyImage) {
-                            docker.image(basePostFeatureLennyImage).inside {
                                 sh "builder/bash/run_entry.sh bases/${baseName} artifacts/bases/${baseName}/post-feature.env post-feature"
                             }
 
-                            docker.image(runtimeImages.lenny).inside {
+                            withImage(env.RUNTIME_LENNY_IMAGE) {
                                 sh "builder/bash/run_profile_build.sh artifacts/profiles/${profileName}/profile-build.env"
                             }
 
-                            withPython {
+                            withBuilder {
                                 sh "python3 builder/py/build.py profile-finalize --profile ${profileName}"
                             }
                         }

builder/py/build.py

@@ -131,43 +131,58 @@ def cmd_list(args: argparse.Namespace) -> int:
     return 0
 
 
-def cmd_module_docker(args: argparse.Namespace) -> int:
+def _module_docker_payload(root: Path, kind: str, name: str, phase: str) -> dict[str, str]:
-    root = root_dir()
+    spec = load_base_spec(root, name) if kind == 'base' else load_feature_spec(root, name)
-    spec = load_base_spec(root, args.name) if args.kind == 'base' else load_feature_spec(root, args.name)
+    stage = spec.docker_overrides.get(phase)
-    stage = spec.docker_overrides.get(args.phase)
+    return {
-    payload = {
         'image': getattr(stage, 'image', '') if stage is not None else '',
         'dockerfile': getattr(stage, 'dockerfile', '') if stage is not None else '',
         'docker_context': getattr(stage, 'docker_context', '') if stage is not None else '',
     }
-    if getattr(args, 'field', None):
+
-        print(payload.get(args.field, ''))
+
-    else:
+def cmd_module_docker(args: argparse.Namespace) -> int:
+    root = root_dir()
+    payload = _module_docker_payload(root, args.kind, args.name, args.phase)
+    if args.json:
         print(json.dumps(payload, sort_keys=True))
+    else:
+        print(payload.get(args.field or '', ''))
     return 0
 
 
-def cmd_profile_info(args: argparse.Namespace) -> int:
+def _profile_resolve_payload(root: Path, profile_name: str) -> dict[str, object]:
-    root = root_dir()
+    profile = load_profile(root, profile_name)
-    profile = load_profile(root, args.profile)
+    return {
-    payload = {
+        'profile': profile_name,
         'base': profile.base,
         'base_chain': [name for name, _spec in load_base_chain(root, profile.base)],
         'features': list(profile.features),
     }
-    if getattr(args, 'field', None):
+
-        if args.field == 'base':
+
-            print(payload['base'])
+def cmd_profile_info(args: argparse.Namespace) -> int:
-        elif args.field == 'base-chain':
+    root = root_dir()
-            for name in payload['base_chain']:
+    payload = _profile_resolve_payload(root, args.profile)
-                print(name)
+    if args.json:
-        elif args.field == 'features':
-            for feature in payload['features']:
-                print(feature)
-        else:
-            raise ValueError(args.field)
-    else:
         print(json.dumps(payload, sort_keys=True))
+        return 0
+    if args.field == 'base':
+        print(payload['base'])
+    elif args.field == 'base-chain':
+        for name in payload['base_chain']:
+            print(name)
+    elif args.field == 'features':
+        for feature in payload['features']:
+            print(feature)
+    else:
+        raise ValueError(args.field)
+    return 0
+
+
+def cmd_profile_resolve(args: argparse.Namespace) -> int:
+    root = root_dir()
+    print(json.dumps(_profile_resolve_payload(root, args.profile), sort_keys=True))
     return 0
 
 
@@ -300,17 +315,19 @@ def build_parser() -> argparse.ArgumentParser:
     p = sub.add_parser('profile-info')
     p.add_argument('--profile', required=True)
     p.add_argument('field', nargs='?', choices=['base', 'base-chain', 'features'])
+    p.add_argument('--json', action='store_true')
     p.set_defaults(func=cmd_profile_info)
 
     p = sub.add_parser('profile-resolve')
     p.add_argument('--profile', required=True)
-    p.set_defaults(func=cmd_profile_info)
+    p.set_defaults(func=cmd_profile_resolve)
 
     p = sub.add_parser('module-docker')
     p.add_argument('--kind', required=True, choices=['base', 'feature'])
     p.add_argument('--name', required=True)
     p.add_argument('--phase', required=True)
     p.add_argument('field', nargs='?', choices=['image', 'dockerfile', 'docker_context'])
+    p.add_argument('--json', action='store_true')
     p.set_defaults(func=cmd_module_docker)
 
     p = sub.add_parser('run-python-phase')