From 5f872b550175e7ea8749ea0ea8503a00b9cf0721 Mon Sep 17 00:00:00 2001 From: chacha Date: Sat, 28 Mar 2026 00:55:29 +0100 Subject: [PATCH] opt --- Jenkinsfile | 204 +++++++++++++++++++++++++------------------- builder/py/build.py | 67 +++++++++------ 2 files changed, 157 insertions(+), 114 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c6568d9..6a471a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,210 +15,236 @@ pipeline { } stages { - stage('Init runtime images and validate builder') { + stage('Build runtimes') { steps { script { - def json = new groovy.json.JsonSlurperClassic() - def runtimeImages = [:] - def moduleDockerCache = [:] - def profileCache = [:] + env.RUNTIME_BUILDER_IMAGE = docker.build( + "retrodebian/builder-alpine:${env.BUILD_TAG}", + "-f ${env.PYTHON_DOCKERFILE} ." + ).imageName() - def buildRuntimeImage = { String runtimeName, String dockerfilePath -> - return docker.build( - "retrodebian/${runtimeName}:${env.BUILD_TAG}", - "-f ${dockerfilePath} ." - ).imageName() + env.RUNTIME_ETCH_IMAGE = docker.build( + "retrodebian/package-builder-etch:${env.BUILD_TAG}", + "-f ${env.ETCH_DOCKERFILE} ." + ).imageName() + + env.RUNTIME_LENNY_IMAGE = docker.build( + "retrodebian/live-helper:${env.BUILD_TAG}", + "-f ${env.LENNY_DOCKERFILE} ." + ).imageName() + } + } + } + + stage('Validate builder') { + steps { + script { + docker.image(env.RUNTIME_BUILDER_IMAGE).inside { + sh 'python3 builder/py/build.py validate' } + } + } + } - runtimeImages.python = buildRuntimeImage('builder-alpine', env.PYTHON_DOCKERFILE) - runtimeImages.etch = buildRuntimeImage('package-builder-etch', env.ETCH_DOCKERFILE) - runtimeImages.lenny = buildRuntimeImage('live-helper', env.LENNY_DOCKERFILE) + stage('Build common features and profiles') { + steps { + script { + def jsonSlurper = new groovy.json.JsonSlurperClassic() + def dockerOverrideCache = [:] + def profileResolveCache = [:] - def withPython = { Closure body -> - docker.image(runtimeImages.python).inside { + def withImage = { String imageName, Closure body -> + docker.image(imageName).inside { body() } } - def shJson = { String scriptText -> - def raw = sh(script: scriptText, returnStdout: true).trim() - return raw ? json.parseText(raw) : [:] + def withBuilder = { Closure body -> + withImage(env.RUNTIME_BUILDER_IMAGE, body) } - def resolveProfile = { String profileName -> - if (!profileCache.containsKey(profileName)) { - withPython { - profileCache[profileName] = shJson("python3 builder/py/build.py profile-resolve --profile ${profileName}") - } + def builderJson = { String command -> + def raw = '' + withBuilder { + raw = sh(script: command, returnStdout: true).trim() } - return profileCache[profileName] - } - - def resolveModuleDocker = { String kind, String name, String phase -> - def cacheKey = "${kind}:${name}:${phase}" - if (!moduleDockerCache.containsKey(cacheKey)) { - withPython { - moduleDockerCache[cacheKey] = shJson( - "python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase}" - ) - } - } - return moduleDockerCache[cacheKey] + return raw ? jsonSlurper.parseText(raw) : [:] } def resolveModuleRuntimeImage = { String runtimeName, String kind, String name, String phase, String defaultImage -> - def overrideCfg = resolveModuleDocker(kind, name, phase) - def overrideImage = (overrideCfg.image ?: '').trim() - def overrideDockerfile = (overrideCfg.dockerfile ?: '').trim() - def overrideContext = (overrideCfg.docker_context ?: '.').trim() + def cacheKey = "${runtimeName}:${kind}:${name}:${phase}" + if (dockerOverrideCache.containsKey(cacheKey)) { + return dockerOverrideCache[cacheKey] + } + def payload = builderJson("python3 builder/py/build.py module-docker --kind ${kind} --name ${name} --phase ${phase} --json") + def overrideImage = (payload.image ?: '').toString().trim() + def overrideDockerfile = (payload.dockerfile ?: '').toString().trim() + def overrideContext = (payload.docker_context ?: '').toString().trim() + + def resolved = defaultImage if (overrideDockerfile) { def tag = overrideImage ? overrideImage : "retrodebian/${runtimeName}-${kind}-${name}-${phase}:${env.BUILD_TAG}" - return docker.build(tag, "-f ${overrideDockerfile} ${overrideContext}").imageName() + def contextPath = overrideContext ? overrideContext : '.' + resolved = docker.build(tag, "-f ${overrideDockerfile} ${contextPath}").imageName() + } else if (overrideImage) { + resolved = overrideImage } - if (overrideImage) { - return overrideImage - } - return defaultImage + + dockerOverrideCache[cacheKey] = resolved + return resolved } - withPython { - sh 'python3 builder/py/build.py validate' + def resolveProfile = { String profileName -> + if (profileResolveCache.containsKey(profileName)) { + return profileResolveCache[profileName] + } + def payload = builderJson("python3 builder/py/build.py profile-resolve --profile ${profileName}") + profileResolveCache[profileName] = payload + return payload } def featureNames = [] - withPython { + withBuilder { def raw = sh(script: 'python3 builder/py/build.py list features', returnStdout: true).trim() featureNames = raw ? raw.split('\n').findAll { it?.trim() } : [] } for (featureName in featureNames) { stage("Feature ${featureName}") { - def pythonPreImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-gen', runtimeImages.python) - docker.image(pythonPreImage).inside { + def preGenPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-gen', env.RUNTIME_BUILDER_IMAGE) + def generateEtchImage = resolveModuleRuntimeImage('etch', 'feature', featureName, 'generate', env.RUNTIME_ETCH_IMAGE) + def postGenPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-gen', env.RUNTIME_BUILDER_IMAGE) + + withImage(preGenPythonImage) { sh "python3 builder/py/build.py run-python-phase --kind feature --phase pre-gen --profile ${params.PROFILE} --feature ${featureName}" sh "python3 builder/py/build.py export-env --kind feature --phase pre-gen --profile ${params.PROFILE} --feature ${featureName} --output artifacts/features/${featureName}/runtime.env" } - def etchImage = resolveModuleRuntimeImage('etch', 'feature', featureName, 'generate', runtimeImages.etch) - docker.image(etchImage).inside { + withImage(generateEtchImage) { sh "builder/bash/run_generate.sh features/${featureName} artifacts/features/${featureName}/runtime.env" } - def pythonPostImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-gen', runtimeImages.python) - docker.image(pythonPostImage).inside { + withImage(postGenPythonImage) { sh "python3 builder/py/build.py run-python-phase --kind feature --phase post-gen --profile ${params.PROFILE} --feature ${featureName}" } } } def profileNames = [] - if (params.BUILD_ALL_PROFILES) { - withPython { + withBuilder { + if (params.BUILD_ALL_PROFILES) { def raw = sh(script: 'python3 builder/py/build.py list profiles', returnStdout: true).trim() profileNames = raw ? raw.split('\n').findAll { it?.trim() } : [] + } else { + profileNames = [params.PROFILE] } - } else { - profileNames = [params.PROFILE] } for (profileName in profileNames) { stage("Profile ${profileName}") { - def profileInfo = resolveProfile(profileName) - def baseName = profileInfo.base - def baseChain = profileInfo.base_chain ?: [] - def featureList = profileInfo.features ?: [] + def profileData = resolveProfile(profileName) + def baseName = profileData.base.toString() + def baseChain = (profileData.base_chain ?: []) as List + def featureList = (profileData.features ?: []) as List for (baseItem in baseChain) { stage("Base ${profileName} / ${baseItem}") { - def pythonPreImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'pre-gen', runtimeImages.python) - docker.image(pythonPreImage).inside { + def preGenPythonImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'pre-gen', env.RUNTIME_BUILDER_IMAGE) + def generateEtchImage = resolveModuleRuntimeImage('etch', 'base', baseItem, 'generate', env.RUNTIME_ETCH_IMAGE) + def postGenPythonImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'post-gen', env.RUNTIME_BUILDER_IMAGE) + + withImage(preGenPythonImage) { sh "python3 builder/py/build.py run-python-phase --kind base --phase pre-gen --profile ${profileName} --base ${baseItem}" sh "python3 builder/py/build.py export-env --kind base --phase pre-gen --profile ${profileName} --base ${baseItem} --output artifacts/bases/${baseItem}/runtime.env" } - def etchImage = resolveModuleRuntimeImage('etch', 'base', baseItem, 'generate', runtimeImages.etch) - docker.image(etchImage).inside { + withImage(generateEtchImage) { sh "builder/bash/run_generate.sh bases/${baseItem} artifacts/bases/${baseItem}/runtime.env" } - def pythonPostImage = resolveModuleRuntimeImage('python', 'base', baseItem, 'post-gen', runtimeImages.python) - docker.image(pythonPostImage).inside { + withImage(postGenPythonImage) { sh "python3 builder/py/build.py run-python-phase --kind base --phase post-gen --profile ${profileName} --base ${baseItem}" } } } - withPython { + withBuilder { sh "python3 builder/py/build.py prepare-profile --profile ${profileName}" sh "python3 builder/py/build.py export-env --kind profile --phase config --profile ${profileName} --output artifacts/profiles/${profileName}/profile-config.env" sh "python3 builder/py/build.py inject-resources --kind base --name ${baseName}" } - docker.image(runtimeImages.lenny).inside { + withImage(env.RUNTIME_LENNY_IMAGE) { sh "builder/bash/run_profile_config.sh artifacts/profiles/${profileName}/profile-config.env" } - def basePreFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'pre-feature', runtimeImages.python) - docker.image(basePreFeaturePythonImage).inside { + def basePreFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'pre-feature', env.RUNTIME_BUILDER_IMAGE) + def basePreFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'pre-feature', env.RUNTIME_LENNY_IMAGE) + + withImage(basePreFeaturePythonImage) { sh "python3 builder/py/build.py run-python-phase --kind base --phase pre-feature --profile ${profileName} --base ${baseName}" sh "python3 builder/py/build.py export-env --kind base --phase pre-feature --profile ${profileName} --base ${baseName} --output artifacts/bases/${baseName}/pre-feature.env" } - def basePreFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'pre-feature', runtimeImages.lenny) - docker.image(basePreFeatureLennyImage).inside { + withImage(basePreFeatureLennyImage) { sh "builder/bash/run_entry.sh bases/${baseName} artifacts/bases/${baseName}/pre-feature.env pre-feature" } for (featureName in featureList) { stage("Inject ${profileName} / ${featureName}") { - def featurePreInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-inj', runtimeImages.python) - docker.image(featurePreInjPythonImage).inside { + def featurePreInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'pre-inj', env.RUNTIME_BUILDER_IMAGE) + def featurePreInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'pre-inj', env.RUNTIME_LENNY_IMAGE) + def featurePostInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'post-inj', env.RUNTIME_LENNY_IMAGE) + def featurePostInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-inj', env.RUNTIME_BUILDER_IMAGE) + + withBuilder { sh "python3 builder/py/build.py save-feature-metadata --profile ${profileName} --feature ${featureName}" + } + + withImage(featurePreInjPythonImage) { sh "python3 builder/py/build.py run-python-phase --kind feature --phase pre-inj --profile ${profileName} --feature ${featureName}" sh "python3 builder/py/build.py export-env --kind feature --phase pre-inj --profile ${profileName} --feature ${featureName} --output artifacts/features/${featureName}/pre-inj.env" } - def featurePreInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'pre-inj', runtimeImages.lenny) - docker.image(featurePreInjLennyImage).inside { + withImage(featurePreInjLennyImage) { sh "builder/bash/run_entry.sh features/${featureName} artifacts/features/${featureName}/pre-inj.env pre-inj" } - def featurePostInjPythonImage = resolveModuleRuntimeImage('python', 'feature', featureName, 'post-inj', runtimeImages.python) - docker.image(featurePostInjPythonImage).inside { + withBuilder { sh "python3 builder/py/build.py inject-resources --kind feature --name ${featureName}" sh "python3 builder/py/build.py export-env --kind feature --phase post-inj --profile ${profileName} --feature ${featureName} --output artifacts/features/${featureName}/post-inj.env" } - def featurePostInjLennyImage = resolveModuleRuntimeImage('lenny', 'feature', featureName, 'post-inj', runtimeImages.lenny) - docker.image(featurePostInjLennyImage).inside { + withImage(featurePostInjLennyImage) { sh "builder/bash/run_entry.sh features/${featureName} artifacts/features/${featureName}/post-inj.env post-inj" } - docker.image(featurePostInjPythonImage).inside { + withImage(featurePostInjPythonImage) { sh "python3 builder/py/build.py run-python-phase --kind feature --phase post-inj --profile ${profileName} --feature ${featureName}" } } } - def basePostFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'post-feature', runtimeImages.python) - docker.image(basePostFeaturePythonImage).inside { + def basePostFeaturePythonImage = resolveModuleRuntimeImage('python', 'base', baseName, 'post-feature', env.RUNTIME_BUILDER_IMAGE) + def basePostFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'post-feature', env.RUNTIME_LENNY_IMAGE) + + withImage(basePostFeaturePythonImage) { sh "python3 builder/py/build.py run-python-phase --kind base --phase post-feature --profile ${profileName} --base ${baseName}" sh "python3 builder/py/build.py export-env --kind base --phase post-feature --profile ${profileName} --base ${baseName} --output artifacts/bases/${baseName}/post-feature.env" sh "python3 builder/py/build.py profile-pre-build --profile ${profileName}" sh "python3 builder/py/build.py export-env --kind profile --phase build --profile ${profileName} --output artifacts/profiles/${profileName}/profile-build.env" } - def basePostFeatureLennyImage = resolveModuleRuntimeImage('lenny', 'base', baseName, 'post-feature', runtimeImages.lenny) - docker.image(basePostFeatureLennyImage).inside { + withImage(basePostFeatureLennyImage) { sh "builder/bash/run_entry.sh bases/${baseName} artifacts/bases/${baseName}/post-feature.env post-feature" } - docker.image(runtimeImages.lenny).inside { + withImage(env.RUNTIME_LENNY_IMAGE) { sh "builder/bash/run_profile_build.sh artifacts/profiles/${profileName}/profile-build.env" } - withPython { + withBuilder { sh "python3 builder/py/build.py profile-finalize --profile ${profileName}" } } diff --git a/builder/py/build.py b/builder/py/build.py index c12032e..9be364a 100644 --- a/builder/py/build.py +++ b/builder/py/build.py @@ -131,43 +131,58 @@ def cmd_list(args: argparse.Namespace) -> int: return 0 -def cmd_module_docker(args: argparse.Namespace) -> int: - root = root_dir() - spec = load_base_spec(root, args.name) if args.kind == 'base' else load_feature_spec(root, args.name) - stage = spec.docker_overrides.get(args.phase) - payload = { +def _module_docker_payload(root: Path, kind: str, name: str, phase: str) -> dict[str, str]: + spec = load_base_spec(root, name) if kind == 'base' else load_feature_spec(root, name) + stage = spec.docker_overrides.get(phase) + return { 'image': getattr(stage, 'image', '') if stage is not None else '', 'dockerfile': getattr(stage, 'dockerfile', '') if stage is not None else '', 'docker_context': getattr(stage, 'docker_context', '') if stage is not None else '', } - if getattr(args, 'field', None): - print(payload.get(args.field, '')) - else: + + +def cmd_module_docker(args: argparse.Namespace) -> int: + root = root_dir() + payload = _module_docker_payload(root, args.kind, args.name, args.phase) + if args.json: print(json.dumps(payload, sort_keys=True)) + else: + print(payload.get(args.field or '', '')) return 0 -def cmd_profile_info(args: argparse.Namespace) -> int: - root = root_dir() - profile = load_profile(root, args.profile) - payload = { +def _profile_resolve_payload(root: Path, profile_name: str) -> dict[str, object]: + profile = load_profile(root, profile_name) + return { + 'profile': profile_name, 'base': profile.base, 'base_chain': [name for name, _spec in load_base_chain(root, profile.base)], 'features': list(profile.features), } - if getattr(args, 'field', None): - if args.field == 'base': - print(payload['base']) - elif args.field == 'base-chain': - for name in payload['base_chain']: - print(name) - elif args.field == 'features': - for feature in payload['features']: - print(feature) - else: - raise ValueError(args.field) - else: + + +def cmd_profile_info(args: argparse.Namespace) -> int: + root = root_dir() + payload = _profile_resolve_payload(root, args.profile) + if args.json: print(json.dumps(payload, sort_keys=True)) + return 0 + if args.field == 'base': + print(payload['base']) + elif args.field == 'base-chain': + for name in payload['base_chain']: + print(name) + elif args.field == 'features': + for feature in payload['features']: + print(feature) + else: + raise ValueError(args.field) + return 0 + + +def cmd_profile_resolve(args: argparse.Namespace) -> int: + root = root_dir() + print(json.dumps(_profile_resolve_payload(root, args.profile), sort_keys=True)) return 0 @@ -300,17 +315,19 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser('profile-info') p.add_argument('--profile', required=True) p.add_argument('field', nargs='?', choices=['base', 'base-chain', 'features']) + p.add_argument('--json', action='store_true') p.set_defaults(func=cmd_profile_info) p = sub.add_parser('profile-resolve') p.add_argument('--profile', required=True) - p.set_defaults(func=cmd_profile_info) + p.set_defaults(func=cmd_profile_resolve) p = sub.add_parser('module-docker') p.add_argument('--kind', required=True, choices=['base', 'feature']) p.add_argument('--name', required=True) p.add_argument('--phase', required=True) p.add_argument('field', nargs='?', choices=['image', 'dockerfile', 'docker_context']) + p.add_argument('--json', action='store_true') p.set_defaults(func=cmd_module_docker) p = sub.add_parser('run-python-phase')