diff --git a/Jenkinsfile b/Jenkinsfile index 7c5b89e..c6568d9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,269 +15,193 @@ pipeline { } stages { - stage('Validate builder') { + stage('Init runtime images and validate builder') { steps { script { - def pythonImage = docker.build( - "retrodebian/builder-alpine:${env.BUILD_TAG}", - "-f ${env.PYTHON_DOCKERFILE} ." - ).imageName() + def json = new groovy.json.JsonSlurperClassic() + def runtimeImages = [:] + def moduleDockerCache = [:] + def profileCache = [:] - docker.image(pythonImage).inside { - sh 'python3 builder/py/build.py validate' - } - } - } - } - - stage('Build common features and profiles') { - steps { - script { - def resolveDefaultRuntimeImage = { String runtimeName, String imageName, String dockerfilePath -> - if (dockerfilePath?.trim()) { - return docker.build( - "retrodebian/${runtimeName}:${env.BUILD_TAG}", - "-f ${dockerfilePath} ." - ).imageName() - } - if (!imageName?.trim()) { - error("Missing default image for runtime ${runtimeName}") - } - return imageName.trim() + def buildRuntimeImage = { String runtimeName, String dockerfilePath -> + return docker.build( + "retrodebian/${runtimeName}:${env.BUILD_TAG}", + "-f ${dockerfilePath} ." + ).imageName() } - def resolveModuleRuntimeImage = { String runtimeName, String kind, String name, String phase, String defaultImage, String pythonControlImage -> - def overrideImage = '' - def overrideDockerfile = '' - def overrideContext = '' + runtimeImages.python = buildRuntimeImage('builder-alpine', env.PYTHON_DOCKERFILE) + runtimeImages.etch = buildRuntimeImage('package-builder-etch', env.ETCH_DOCKERFILE) + runtimeImages.lenny = buildRuntimeImage('live-helper', env.LENNY_DOCKERFILE) - docker.image(pythonControlImage).inside { - overrideImage = sh( - script: "python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase} image", - returnStdout: true - ).trim() - - overrideDockerfile = sh( - script: "python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase} dockerfile", - returnStdout: true - ).trim() - - overrideContext = sh( - script: "python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase} docker_context", - returnStdout: true - ).trim() + def withPython = { Closure body -> + docker.image(runtimeImages.python).inside { + body() } + } + + def shJson = { String scriptText -> + def raw = sh(script: scriptText, returnStdout: true).trim() + return raw ? json.parseText(raw) : [:] + } + + def resolveProfile = { String profileName -> + if (!profileCache.containsKey(profileName)) { + withPython { + profileCache[profileName] = shJson("python3 builder/py/build.py profile-resolve --profile ${profileName}") + } + } + return profileCache[profileName] + } + + def resolveModuleDocker = { String kind, String name, String phase -> + def cacheKey = "${kind}:${name}:${phase}" + if (!moduleDockerCache.containsKey(cacheKey)) { + withPython { + moduleDockerCache[cacheKey] = shJson( + "python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase}" + ) + } + } + return moduleDockerCache[cacheKey] + } + + def resolveModuleRuntimeImage = { String runtimeName, String kind, String name, String phase, String defaultImage -> + def overrideCfg = resolveModuleDocker(kind, name, phase) + def overrideImage = (overrideCfg.image ?: '').trim() + def overrideDockerfile = (overrideCfg.dockerfile ?: '').trim() + def overrideContext = (overrideCfg.docker_context ?: '.').trim() if (overrideDockerfile) { def tag = overrideImage ? overrideImage : "retrodebian/${runtimeName}-${kind}-${name}-${phase}:${env.BUILD_TAG}" - def contextPath = overrideContext ? overrideContext : '.' - return docker.build(tag, "-f ${overrideDockerfile} ${contextPath}").imageName() + return docker.build(tag, "-f ${overrideDockerfile} ${overrideContext}").imageName() } - if (overrideImage) { return overrideImage } - return defaultImage } - def pythonDefaultImage = resolveDefaultRuntimeImage( - 'builder-alpine', - '', - env.PYTHON_DOCKERFILE - ) - - def etchDefaultImage = resolveDefaultRuntimeImage( - 'package-builder-etch', - '', - env.ETCH_DOCKERFILE - ) - - def lennyDefaultImage = resolveDefaultRuntimeImage( - 'live-helper', - '', - env.LENNY_DOCKERFILE - ) + withPython { + sh 'python3 builder/py/build.py validate' + } def featureNames = [] - docker.image(pythonDefaultImage).inside { + withPython { def raw = sh(script: 'python3 builder/py/build.py list features', returnStdout: true).trim() featureNames = raw ? raw.split('\n').findAll { it?.trim() } : [] } for (featureName in featureNames) { stage("Feature ${featureName}") { - def pythonImagePre = resolveModuleRuntimeImage( - 'python', 'feature', featureName, 'pre-gen', - pythonDefaultImage, pythonDefaultImage - ) - docker.image(pythonImagePre).inside { + def pythonPreImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-gen', runtimeImages.python) + docker.image(pythonPreImage).inside { sh "python3 builder/py/build.py run-python-phase --kind feature --phase pre-gen --profile ${params.PROFILE} --feature ${featureName}" sh "python3 builder/py/build.py export-env --kind feature --phase pre-gen --profile ${params.PROFILE} --feature ${featureName} --output artifacts/features/${featureName}/runtime.env" } - def etchImage = resolveModuleRuntimeImage( - 'etch', 'feature', featureName, 'generate', - etchDefaultImage, pythonDefaultImage - ) + def etchImage = resolveModuleRuntimeImage('etch', 'feature', featureName, 'generate', runtimeImages.etch) docker.image(etchImage).inside { sh "builder/bash/run_generate.sh features/${featureName} artifacts/features/${featureName}/runtime.env" } - def pythonImagePost = resolveModuleRuntimeImage( - 'python', 'feature', featureName, 'post-gen', - pythonDefaultImage, pythonDefaultImage - ) - docker.image(pythonImagePost).inside { + def pythonPostImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-gen', runtimeImages.python) + docker.image(pythonPostImage).inside { sh "python3 builder/py/build.py run-python-phase --kind feature --phase post-gen --profile ${params.PROFILE} --feature ${featureName}" } } } def profileNames = [] - docker.image(pythonDefaultImage).inside { - if (params.BUILD_ALL_PROFILES) { + if (params.BUILD_ALL_PROFILES) { + withPython { def raw = sh(script: 'python3 builder/py/build.py list profiles', returnStdout: true).trim() profileNames = raw ? raw.split('\n').findAll { it?.trim() } : [] - } else { - profileNames = [params.PROFILE] } + } else { + profileNames = [params.PROFILE] } for (profileName in profileNames) { stage("Profile ${profileName}") { - def baseName = '' - def baseChain = [] - def featureList = [] - - docker.image(pythonDefaultImage).inside { - baseName = sh( - script: "python3 builder/py/build.py profile-info --profile ${profileName} base", - returnStdout: true - ).trim() - - def chainRaw = sh( - script: "python3 builder/py/build.py profile-info --profile ${profileName} base-chain", - returnStdout: true - ).trim() - baseChain = chainRaw ? chainRaw.split('\n').findAll { it?.trim() } : [] - - def featuresRaw = sh( - script: "python3 builder/py/build.py profile-info --profile ${profileName} features", - returnStdout: true - ).trim() - featureList = featuresRaw ? featuresRaw.split('\n').findAll { it?.trim() } : [] - } + def profileInfo = resolveProfile(profileName) + def baseName = profileInfo.base + def baseChain = profileInfo.base_chain ?: [] + def featureList = profileInfo.features ?: [] for (baseItem in baseChain) { stage("Base ${profileName} / ${baseItem}") { - def pythonImagePre = resolveModuleRuntimeImage( - 'python', 'base', baseItem, 'pre-gen', - pythonDefaultImage, pythonDefaultImage - ) - docker.image(pythonImagePre).inside { + def pythonPreImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'pre-gen', runtimeImages.python) + docker.image(pythonPreImage).inside { sh "python3 builder/py/build.py run-python-phase --kind base --phase pre-gen --profile ${profileName} --base ${baseItem}" sh "python3 builder/py/build.py export-env --kind base --phase pre-gen --profile ${profileName} --base ${baseItem} --output artifacts/bases/${baseItem}/runtime.env" } - def etchImage = resolveModuleRuntimeImage( - 'etch', 'base', baseItem, 'generate', - etchDefaultImage, pythonDefaultImage - ) + def etchImage = resolveModuleRuntimeImage('etch', 'base', baseItem, 'generate', runtimeImages.etch) docker.image(etchImage).inside { sh "builder/bash/run_generate.sh bases/${baseItem} artifacts/bases/${baseItem}/runtime.env" } - def pythonImagePost = resolveModuleRuntimeImage( - 'python', 'base', baseItem, 'post-gen', - pythonDefaultImage, pythonDefaultImage - ) - docker.image(pythonImagePost).inside { + def pythonPostImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'post-gen', runtimeImages.python) + docker.image(pythonPostImage).inside { sh "python3 builder/py/build.py run-python-phase --kind base --phase post-gen --profile ${profileName} --base ${baseItem}" } } } - docker.image(pythonDefaultImage).inside { + withPython { sh "python3 builder/py/build.py prepare-profile --profile ${profileName}" sh "python3 builder/py/build.py export-env --kind profile --phase config --profile ${profileName} --output artifacts/profiles/${profileName}/profile-config.env" - } - - docker.image(lennyDefaultImage).inside { - sh "builder/bash/run_profile_config.sh artifacts/profiles/${profileName}/profile-config.env" - } - - docker.image(pythonDefaultImage).inside { sh "python3 builder/py/build.py inject-resources --kind base --name ${baseName}" } - def basePreFeaturePythonImage = resolveModuleRuntimeImage( - 'python', 'base', baseName, 'pre-feature', - pythonDefaultImage, pythonDefaultImage - ) + docker.image(runtimeImages.lenny).inside { + sh "builder/bash/run_profile_config.sh artifacts/profiles/${profileName}/profile-config.env" + } + + def basePreFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'pre-feature', runtimeImages.python) docker.image(basePreFeaturePythonImage).inside { sh "python3 builder/py/build.py run-python-phase --kind base --phase pre-feature --profile ${profileName} --base ${baseName}" sh "python3 builder/py/build.py export-env --kind base --phase pre-feature --profile ${profileName} --base ${baseName} --output artifacts/bases/${baseName}/pre-feature.env" } - def basePreFeatureLennyImage = resolveModuleRuntimeImage( - 'lenny', 'base', baseName, 'pre-feature', - lennyDefaultImage, pythonDefaultImage - ) + def basePreFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'pre-feature', runtimeImages.lenny) docker.image(basePreFeatureLennyImage).inside { sh "builder/bash/run_entry.sh bases/${baseName} artifacts/bases/${baseName}/pre-feature.env pre-feature" } for (featureName in featureList) { stage("Inject ${profileName} / ${featureName}") { - docker.image(pythonDefaultImage).inside { - sh "python3 builder/py/build.py save-feature-metadata --profile ${profileName} --feature ${featureName}" - } - - def featurePreInjPythonImage = resolveModuleRuntimeImage( - 'python', 'feature', featureName, 'pre-inj', - pythonDefaultImage, pythonDefaultImage - ) + def featurePreInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-inj', runtimeImages.python) docker.image(featurePreInjPythonImage).inside { + sh "python3 builder/py/build.py save-feature-metadata --profile ${profileName} --feature ${featureName}" sh "python3 builder/py/build.py run-python-phase --kind feature --phase pre-inj --profile ${profileName} --feature ${featureName}" sh "python3 builder/py/build.py export-env --kind feature --phase pre-inj --profile ${profileName} --feature ${featureName} --output artifacts/features/${featureName}/pre-inj.env" } - def featurePreInjLennyImage = resolveModuleRuntimeImage( - 'lenny', 'feature', featureName, 'pre-inj', - lennyDefaultImage, pythonDefaultImage - ) + def featurePreInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'pre-inj', runtimeImages.lenny) docker.image(featurePreInjLennyImage).inside { sh "builder/bash/run_entry.sh features/${featureName} artifacts/features/${featureName}/pre-inj.env pre-inj" } - docker.image(pythonDefaultImage).inside { + def featurePostInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-inj', runtimeImages.python) + docker.image(featurePostInjPythonImage).inside { sh "python3 builder/py/build.py inject-resources --kind feature --name ${featureName}" sh "python3 builder/py/build.py export-env --kind feature --phase post-inj --profile ${profileName} --feature ${featureName} --output artifacts/features/${featureName}/post-inj.env" } - def featurePostInjLennyImage = resolveModuleRuntimeImage( - 'lenny', 'feature', featureName, 'post-inj', - lennyDefaultImage, pythonDefaultImage - ) + def featurePostInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'post-inj', runtimeImages.lenny) docker.image(featurePostInjLennyImage).inside { sh "builder/bash/run_entry.sh features/${featureName} artifacts/features/${featureName}/post-inj.env post-inj" } - def featurePostInjPythonImage = resolveModuleRuntimeImage( - 'python', 'feature', featureName, 'post-inj', - pythonDefaultImage, pythonDefaultImage - ) docker.image(featurePostInjPythonImage).inside { sh "python3 builder/py/build.py run-python-phase --kind feature --phase post-inj --profile ${profileName} --feature ${featureName}" } } } - def basePostFeaturePythonImage = resolveModuleRuntimeImage( - 'python', 'base', baseName, 'post-feature', - pythonDefaultImage, pythonDefaultImage - ) + def basePostFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'post-feature', runtimeImages.python) docker.image(basePostFeaturePythonImage).inside { sh "python3 builder/py/build.py run-python-phase --kind base --phase post-feature --profile ${profileName} --base ${baseName}" sh "python3 builder/py/build.py export-env --kind base --phase post-feature --profile ${profileName} --base ${baseName} --output artifacts/bases/${baseName}/post-feature.env" @@ -285,19 +209,16 @@ pipeline { sh "python3 builder/py/build.py export-env --kind profile --phase build --profile ${profileName} --output artifacts/profiles/${profileName}/profile-build.env" } - def basePostFeatureLennyImage = resolveModuleRuntimeImage( - 'lenny', 'base', baseName, 'post-feature', - lennyDefaultImage, pythonDefaultImage - ) + def basePostFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'post-feature', runtimeImages.lenny) docker.image(basePostFeatureLennyImage).inside { sh "builder/bash/run_entry.sh bases/${baseName} artifacts/bases/${baseName}/post-feature.env post-feature" } - docker.image(lennyDefaultImage).inside { + docker.image(runtimeImages.lenny).inside { sh "builder/bash/run_profile_build.sh artifacts/profiles/${profileName}/profile-build.env" } - docker.image(pythonDefaultImage).inside { + withPython { sh "python3 builder/py/build.py profile-finalize --profile ${profileName}" } } @@ -313,4 +234,4 @@ pipeline { archiveArtifacts artifacts: 'live/**/*', allowEmptyArchive: true } } -} \ No newline at end of file +} diff --git a/builder/py/build.py b/builder/py/build.py index f5bfc9c..c12032e 100644 --- a/builder/py/build.py +++ b/builder/py/build.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import json import os import subprocess import sys @@ -134,23 +135,39 @@ def cmd_module_docker(args: argparse.Namespace) -> int: root = root_dir() spec = load_base_spec(root, args.name) if args.kind == 'base' else load_feature_spec(root, args.name) stage = spec.docker_overrides.get(args.phase) - print(getattr(stage, args.field) if stage is not None and getattr(stage, args.field) else '') + payload = { + 'image': getattr(stage, 'image', '') if stage is not None else '', + 'dockerfile': getattr(stage, 'dockerfile', '') if stage is not None else '', + 'docker_context': getattr(stage, 'docker_context', '') if stage is not None else '', + } + if getattr(args, 'field', None): + print(payload.get(args.field, '')) + else: + print(json.dumps(payload, sort_keys=True)) return 0 def cmd_profile_info(args: argparse.Namespace) -> int: root = root_dir() profile = load_profile(root, args.profile) - if args.field == 'base': - print(profile.base) - elif args.field == 'base-chain': - for name, _spec in load_base_chain(root, profile.base): - print(name) - elif args.field == 'features': - for feature in profile.features: - print(feature) + payload = { + 'base': profile.base, + 'base_chain': [name for name, _spec in load_base_chain(root, profile.base)], + 'features': list(profile.features), + } + if getattr(args, 'field', None): + if args.field == 'base': + print(payload['base']) + elif args.field == 'base-chain': + for name in payload['base_chain']: + print(name) + elif args.field == 'features': + for feature in payload['features']: + print(feature) + else: + raise ValueError(args.field) else: - raise ValueError(args.field) + print(json.dumps(payload, sort_keys=True)) return 0 @@ -282,14 +299,18 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser('profile-info') p.add_argument('--profile', required=True) - p.add_argument('field', choices=['base', 'base-chain', 'features']) + p.add_argument('field', nargs='?', choices=['base', 'base-chain', 'features']) + p.set_defaults(func=cmd_profile_info) + + p = sub.add_parser('profile-resolve') + p.add_argument('--profile', required=True) p.set_defaults(func=cmd_profile_info) p = sub.add_parser('module-docker') p.add_argument('--kind', required=True, choices=['base', 'feature']) p.add_argument('--name', required=True) p.add_argument('--phase', required=True) - p.add_argument('field', choices=['image', 'dockerfile', 'docker_context']) + p.add_argument('field', nargs='?', choices=['image', 'dockerfile', 'docker_context']) p.set_defaults(func=cmd_module_docker) p = sub.add_parser('run-python-phase')