From 18e9063895a7d171181018545e7331212892cde4 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 4 Sep 2025 19:38:07 +0200 Subject: [PATCH 1/3] New subcommand 'container builddep' This just downloads the build dependencies that would be installed, with the intent to use this as a cache in a later commit. Signed-off-by: Yann Dirson --- container/files/init-container.sh | 54 +++++++++++++++++++------------ src/xcp_ng_dev/cli.py | 27 ++++++++++++++++ 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/container/files/init-container.sh b/container/files/init-container.sh index 27f8ef0..4f67979 100755 --- a/container/files/init-container.sh +++ b/container/files/init-container.sh @@ -69,10 +69,12 @@ ulimit -s 16384 # get the package arch used in the container (eg. "x86_64_v2") RPMARCH=$(rpm -q glibc --qf "%{arch}") -if [ -n "$BUILD_LOCAL" ]; then +if [ -n "$BUILD_LOCAL$BUILD_DEPS" ]; then time ( cd ~/rpmbuild - rm BUILD BUILDROOT RPMS SRPMS -rf + if [ -n "$BUILD_LOCAL" ]; then + rm BUILD BUILDROOT RPMS SRPMS -rf + fi if specs=$(ls *.spec 2>/dev/null); then SPECFLAGS=( @@ -88,28 +90,40 @@ if [ -n "$BUILD_LOCAL" ]; then case "$OS_RELEASE" in 8.2.*|8.3.*) ;; # sources always available via git-lfs - 8.99.*|9.*) if [ -r sources ]; then alma_get_sources -i sources; fi ;; + 8.99.*|9.*) + if [ -r sources ]; then alma_get_sources -i sources; fi + # FIXME: if [ -n "$BUILD_DEPS" ]; then exfiltrate alma_get_sources + ;; *) echo >&2 "ERROR: unknown release, cannot know package manager"; exit 1 ;; esac - sudo $BDEP "${SPECFLAGS[@]}" -y $specs - - : ${RPMBUILD_STAGE:=a} # default if not specified: -ba - RPMBUILDFLAGS=( - -b${RPMBUILD_STAGE} $specs - --target "$RPMARCH" - $RPMBUILD_OPTS - "${SPECFLAGS[@]}" - ) - # in case the build deps contain xs-opam-repo, source the added profile.d file - [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh - if [ $? == 0 ]; then - if [ -n "$RPMBUILD_DEFINE" ]; then - RPMBUILDFLAGS+=(--define "$RPMBUILD_DEFINE") + if [ -n "$BUILD_DEPS" ]; then + BDEPFLAGS=() + if [ -d ~/builddep/ ]; then + BDEPFLAGS+=(--downloaddir ~/builddep/) fi - rpmbuild "${RPMBUILDFLAGS[@]}" - if [ $? == 0 -a -d ~/output/ ]; then - cp -rf RPMS SRPMS ~/output/ + sudo $BDEP "${SPECFLAGS[@]}" --downloadonly "${BDEPFLAGS[@]}" -y $specs + fi + + if [ -n "$BUILD_LOCAL" ]; then + sudo $BDEP "${SPECFLAGS[@]}" -y $specs + : ${RPMBUILD_STAGE:=a} # default if not specified: -ba + RPMBUILDFLAGS=( + -b${RPMBUILD_STAGE} $specs + --target "$RPMARCH" + $RPMBUILD_OPTS + "${SPECFLAGS[@]}" + ) + # in case the build deps contain xs-opam-repo, source the added profile.d file + [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh + if [ $? == 0 ]; then + if [ -n "$RPMBUILD_DEFINE" ]; then + RPMBUILDFLAGS+=(--define "$RPMBUILD_DEFINE") + fi + rpmbuild "${RPMBUILDFLAGS[@]}" + if [ $? == 0 -a -d ~/output/ ]; then + cp -rf RPMS SRPMS ~/output/ + fi fi fi ) diff --git a/src/xcp_ng_dev/cli.py b/src/xcp_ng_dev/cli.py index 6b0d072..2fe7992 100755 --- a/src/xcp_ng_dev/cli.py +++ b/src/xcp_ng_dev/cli.py @@ -119,6 +119,23 @@ def buildparser(): '--rpmbuild-stage', action='store', help=f"Request given -bX stage rpmbuild, X in [{RPMBUILD_STAGES}]") + # builddep -- fetch/cache builddep of an rpm using a container + parser_builddep = subparsers_container.add_parser( + 'builddep', + help="Fetch dependencies for the spec file(s) found in the SPECS/ subdirectory " + "of the directory passed as parameter.") + add_container_args(parser_builddep) + add_common_args(parser_builddep) + group_builddep = parser_builddep.add_argument_group("builddep arguments") + group_builddep.add_argument( + 'builddep_dir', + help="Directory where the build-dependency RPMs will be cached. " + "The directory is created if it doesn't exist") + group_builddep.add_argument( + 'source_dir', nargs='?', default='.', + help="Root path where SPECS/ and SOURCES are available. " + "The default is the working directory") + # run -- execute commands inside a container parser_run = subparsers_container.add_parser( 'run', @@ -226,6 +243,16 @@ def container(args): docker_args += ["-e", "BUILD_LOCAL=1"] print(f"Building directory {build_dir}", file=sys.stderr) + case 'builddep': + build_dir = os.path.abspath(args.source_dir) + docker_args += ["-v", f"{build_dir}:/home/builder/rpmbuild"] + docker_args += ["-e", "BUILD_DEPS=1"] + + if args.builddep_dir: + os.makedirs(args.builddep_dir, exist_ok=True) + docker_args += ["-v", "%s:/home/builder/builddep:rw" % + os.path.abspath(args.builddep_dir)] + case 'run': docker_args += ["-e", "COMMAND=%s" % ' '.join(args.command)] From 6f444e03cfc8b495a18565a267aa0051969a7f52 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 5 Sep 2025 10:27:36 +0200 Subject: [PATCH 2/3] WIP provide a --local-repo flag Set all local repos to priority 1, to avoid getting e.g. ocaml5 from Alma. FIXME: this curently assumes the "local repos" are plain dirs, and we run createrepo_c on them. This allows to use the output of "builddep" as a cache without any other intermediate step, but that feels awkward for the general case. Maybe we should drop it? Anyway the "builddep" operation may change... --- src/xcp_ng_dev/cli.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/xcp_ng_dev/cli.py b/src/xcp_ng_dev/cli.py index 2fe7992..9c57144 100755 --- a/src/xcp_ng_dev/cli.py +++ b/src/xcp_ng_dev/cli.py @@ -53,6 +53,8 @@ def add_common_args(parser): group.add_argument('--disablerepo', help='disable repositories. Same syntax as yum\'s --disablerepo parameter. ' 'If both --enablerepo and --disablerepo are set, --disablerepo will be applied first') + group.add_argument('--local-repo', action='append', default=[], + help="Directory where the build-dependency RPMs will be taken from.") group.add_argument('--no-update', action='store_true', help='do not run "yum update" on container start, use it as it was at build time') @@ -159,6 +161,24 @@ def buildparser(): return parser +def _setup_repo(repo_dir, name, docker_args): + subprocess.check_call(["createrepo_c", "--compatibility", repo_dir]) + outer_path = os.path.abspath(repo_dir) + inner_path = f"/home/builder/local-repos/{name}" + docker_args += ["-v", f"{outer_path}:{inner_path}:ro" ] + with open(os.path.join(repo_dir, "builddep.repo"), "wt") as repofd: + repofd.write(f""" +[{name}] +name=Local repository - {name} from {outer_path} +baseurl=file:///home/builder/local-repos/{name}/ +enabled=1 +repo_gpgcheck=0 +gpgcheck=0 +priority=1 + """) + # need rw for --disablerepo=* --enablerepo=builddeb + docker_args += ["-v", f"{outer_path}/builddep.repo:/etc/yum.repos.d/{name}.repo:rw"] + def container(args): docker_args = [RUNNER, "run", "-i", "-t"] @@ -221,6 +241,10 @@ def container(args): if args.debug: docker_args += ["-e", "SCRIPT_DEBUG=1"] + for repo in args.local_repo: + # FIXME: ensure name is unique + _setup_repo(repo, os.path.basename(repo), docker_args) + # action-specific match args.action: case 'build': From 28cf9abcb15246377a68057cbde72208808d886b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 11 Sep 2025 15:49:02 +0200 Subject: [PATCH 3/3] Add --no-network option Completely disables network inside the container, making sure we fully control the supply chain. Signed-off-by: Yann Dirson --- src/xcp_ng_dev/cli.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/xcp_ng_dev/cli.py b/src/xcp_ng_dev/cli.py index 9c57144..4d0c0a3 100755 --- a/src/xcp_ng_dev/cli.py +++ b/src/xcp_ng_dev/cli.py @@ -57,6 +57,8 @@ def add_common_args(parser): help="Directory where the build-dependency RPMs will be taken from.") group.add_argument('--no-update', action='store_true', help='do not run "yum update" on container start, use it as it was at build time') + group.add_argument('--no-network', action='store_true', + help='disable all networking support in the build environment') def add_container_args(parser): group = parser.add_argument_group("container arguments") @@ -212,6 +214,11 @@ def container(args): docker_args += ["-e", "DISABLEREPO=%s" % args.disablerepo] if args.no_update: docker_args += ["-e", "NOUPDATE=1"] + if args.no_network: + docker_args += ["--network", "none"] + + if args.no_network and not args.no_update: + print("WARNING: network disabled but --no-update not passed", file=sys.stderr) # container args if args.volume: @@ -248,6 +255,9 @@ def container(args): # action-specific match args.action: case 'build': + if args.no_network and not args.local_repo: + print("WARNING: network disabled but --local-repo not passed", file=sys.stderr) + build_dir = os.path.abspath(args.source_dir) if args.define: docker_args += ["-e", "RPMBUILD_DEFINE=%s" % args.define]