Added numbered symlinks to build steps for ease of use

README-build-in-container.md

@@ -0,0 +1,8 @@
+Build with ./build-in-container.py
+
+Using ./build-in-container.py --shell you can run the build interactively and debug issues.
+Start in the container by running /srv/source/buildscripts/build-in-container-inner.sh
+
+This will copy repository sources that are needed from the read-only /srv location to read-write work area in /home/builder/build.
+
+Continue debugging by running steps in /home/builder/buildscripts/build-scripts/0*.sh

build-in-container-inner.sh

@@ -4,15 +4,20 @@ set -e
 # Configuration via environment variables:
 #   PROJECT, BUILD_TYPE, EXPLICIT_ROLE, BUILD_NUMBER, EXPLICIT_VERSION
 
+# let setup-cfengine-build-host.sh know we are in a container
+sudo touch /etc/cfengine-in-container.flag
+
 BASEDIR=/home/builder/build
 export BASEDIR
 export AUTOBUILD_PATH="$BASEDIR/buildscripts"
+OUTPUT=/output
+export OUTPUT
 
 mkdir -p "$BASEDIR"
 
 # Bind-mounted directories may be owned by the host user's UID.
 # Fix ownership so builder can write to them.
-sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" /output
+sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" "$OUTPUT"
 
 # And hand ownership back to the host user on the way out.
 if [ -n "$HOST_UID" ] && [ -n "$HOST_GID" ]; then
@@ -85,105 +90,11 @@ fi
 export SOURCE_DATE_EPOCH
 echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 
-install_mission_portal_deps() (
-    set -e
-
-    if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then
-        echo "Installing npm dependencies..."
-        npm ci --prefix "$BASEDIR/mission-portal/public/scripts/"
-        echo "Building react components..."
-        npm run build --prefix "$BASEDIR/mission-portal/public/scripts/"
-        rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules"
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/composer.json" ]; then
-        echo "Installing Mission Portal PHP dependencies..."
-        (cd "$BASEDIR/mission-portal" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then
-        echo "Installing Nova API PHP dependencies..."
-        (cd "$BASEDIR/nova/api/http" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then
-        echo "Compiling Mission Portal styles..."
-        mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css"
-        (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" &&
-            lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css)
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then
-        echo "Installing LDAP API PHP dependencies..."
-        (cd "$BASEDIR/mission-portal/ldap" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    # Composer falls back to git clone when GitHub's anonymous zipball
-    # rate limit is hit, leaving non-reproducible .git directories in the
-    # vendor tree. Strip them.
-    find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} +
-)
-
-# Lets whoever consumes the output check that it arrived intact. Sorted in the C
-# locale so that the list itself comes out the same every time.
-write_sha256sums() (
-    cd /output
-    # shellcheck disable=SC2094
-    # > Make sure not to read and write the same file in the same pipeline.
-    # find leaves it out by name, so the list never covers itself.
-    find . -maxdepth 1 -type f ! -name sha256sums.txt -printf '%P\n' \
-        | LC_ALL=C sort | xargs -r sha256sum > sha256sums.txt
-)
-
-# Build the source tarballs. They are the same whichever platform builds them,
-# so only this image builds them, and nothing else here does. /output is
-# <output-dir>/tarballs on the host, as the packages' /output is per label.
-#
-# Each tarball's timestamps follow its own repository: Makefile.am in core and in
-# masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking
-# it from the last commit keeps a tarball identical until its own sources change.
-build_tarballs() (
-    set -e
-
-    (
-        cd "$BASEDIR/core"
-        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
-        export SOURCE_DATE_EPOCH
-        echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
-        rm -f cfengine-3.*.tar.gz
-        # Configure so the dist target exists, undone again below.
-        ./configure -C
-        make dist
-        mv cfengine-3.*.tar.gz /output/
-        make distclean
-    )
-
-    (
-        cd "$BASEDIR/masterfiles"
-        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
-        export SOURCE_DATE_EPOCH
-        echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
-        rm -f cfengine-masterfiles*.tar.gz
-        ./configure
-        make dist        # source tarball:  cfengine-masterfiles-<version>.tar.gz
-        make tar-package # package tarball: cfengine-masterfiles-<version>.pkg.tar.gz
-        mv cfengine-masterfiles*.tar.gz /output/
-        make distclean
-    )
-
-    write_sha256sums
-)
-
-# === Step runner with failure reporting ===
-# Disable set -e so we can capture exit codes and report which step failed.
-set +e
 run_step() {
     local name="$1"
     shift
     echo "=== Running $name ==="
-    "$@"
+    "$BASEDIR/buildscripts/build-scripts/$name" "$@"
     local rc=$?
     if [ $rc -ne 0 ]; then
         echo ""
@@ -193,36 +104,27 @@ run_step() {
 }
 
 # === Build steps ===
-run_step "01-autogen" "$BASEDIR/buildscripts/build-scripts/autogen"
 
 if [ "$TARBALLS" = yes ]; then
-    run_step "02-tarballs" build_tarballs
+    run_step autogen
+    run_step generate-pull-request-file
+    run_step build-tarballs
+    run_step generate-checksum-list
     echo ""
     echo "=== Build complete ==="
-    ls -lh /output/
+    ls -lh "$OUTPUT"
     exit 0
 fi
 
-run_step "02-install-dependencies" "$BASEDIR/buildscripts/build-scripts/install-dependencies"
-# Mission Portal is an Enterprise/nova-only component; its sources are only
-# synced when PROJECT=nova. Skip this step for community hubs.
-if [ "$PROJECT" = "nova" ] && [ "$EXPLICIT_ROLE" = "hub" ]; then
-    run_step "03-mission-portal-deps" install_mission_portal_deps
-fi
-run_step "04-configure" "$BASEDIR/buildscripts/build-scripts/configure"
-run_step "05-compile" "$BASEDIR/buildscripts/build-scripts/compile"
-run_step "06-package" "$BASEDIR/buildscripts/build-scripts/package"
+NO_TESTS=true
+export NO_TESTS
 
-# === Copy output packages ===
-# Packages are created under $BASEDIR/<project>/ by dpkg-buildpackage / rpmbuild.
-# Exclude deps-packaging to avoid copying dependency packages.
-find "$BASEDIR" -maxdepth 4 \
-    -path "$BASEDIR/buildscripts/deps-packaging" -prune -o \
-    \( -name '*.deb' -o -name '*.rpm' -o -name '*.msi' -o -name '*.pkg.tar.gz' \) -print \
-    -exec cp {} /output/ \;
+for script in "$BASEDIR/buildscripts/build-scripts"/0*; do
+  name="$(basename "$script")"
+  run_step "$name"
+done
 
-write_sha256sums
 
 echo ""
 echo "=== Build complete ==="
-ls -lh /output/
+ls -lhR "$OUTPUT"/

build-in-container.py

@@ -406,8 +406,7 @@ def run_container(args, image_tag, source_dir, script_dir, label):
     # Keep the packages in a directory of their own, so that building several
     # platforms into one output directory does not mix them together. The
     # tarballs belong to no platform, so they sit beside those directories.
-    subdir = "tarballs" if args.tarballs else label
-    output_dir = Path(args.output_dir).resolve() / subdir
+    output_dir = Path(args.output_dir).resolve()
     cache_dir = Path(args.cache_dir).resolve()
 
     # Start from an empty directory. An earlier build's packages carry their own
@@ -625,6 +624,21 @@ def parse_args():
         args.platform, args.role, args.arch = found
         log.info(f"{args.label}: {args.platform} {args.role} {args.arch}")
 
+    # Before we run any builds including tarballs we need to validate the platform, project and role
+    if args.project == "community":
+        if args.platform is not None and args.platform.find("HUB") > 0:
+            parser.error(f"--project {args.project} cannot have a hub platform like --platform {args.platform}")
+            sys.exit(0)
+        if args.role != "agent":
+            parser.error(f"--project {args.project} must have agent role. Got --role {args.role}")
+            sys.exit(0)
+
+    if args.platform is not None and args.platform.find("HUB") > 0:
+        if args.project != "nova":
+            parser.error(f"--platform {args.platform} is a hub platform and requires --project nova")
+        if args.role != "hub":
+            parser.error(f"--platform {args.platform} is a hub platform and requires --role hub")
+
     if args.tarballs:
         # The tarballs are built from core and masterfiles alone, in an image of
         # their own, so the platform, project and role are not choices here. The

build-scripts/0000-system-dependencies

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -ex
+thisdir="$(dirname "$0")"
+sudo bash "$thisdir"/../ci/setup-cfengine-build-host.sh

build-scripts/0005-repositories

@@ -0,0 +1 @@
+repositories

build-scripts/0010-autogen

@@ -0,0 +1 @@
+autogen

build-scripts/0020-clean-buildmachine

@@ -0,0 +1 @@
+clean-buildmachine

build-scripts/0030-bootstrap-mission-portal

@@ -0,0 +1 @@
+bootstrap-mission-portal

build-scripts/0035-generate-pull-request-file

@@ -0,0 +1 @@
+generate-pull-request-file

build-scripts/0040-build-tarballs

@@ -0,0 +1 @@
+build-tarballs

build-scripts/0050-unpack-tarballs

@@ -0,0 +1 @@
+unpack-tarballs

build-scripts/0060-install-dependencies

@@ -0,0 +1 @@
+install-dependencies

build-scripts/0070-configure

@@ -0,0 +1 @@
+configure

build-scripts/0080-generate-source-tarballs

@@ -0,0 +1 @@
+generate-source-tarballs

build-scripts/0090-compile

@@ -0,0 +1 @@
+compile

build-scripts/0100-produce-debug-symbols

@@ -0,0 +1 @@
+produce-debug-symbols

build-scripts/0110-package

@@ -0,0 +1 @@
+package

build-scripts/0120-prepare-results

@@ -0,0 +1 @@
+prepare-results

build-scripts/0130-test

@@ -0,0 +1 @@
+test

build-scripts/0140-prepare-results

@@ -0,0 +1 @@
+prepare-results

build-scripts/0150-generate-checksum-list

@@ -0,0 +1 @@
+generate-checksum-list

build-scripts/autogen

@@ -18,6 +18,10 @@
 # ```
 #
 
+# note that this script should be a no-op if an artifacts.tgz is present
+thisdir="$(dirname "$0")"
+bash "$thisdir"/../ci/setup-ci-host.sh --bootstrap
+
 # Get the BASEDIR variable holding the path to where our repos are checked out
 . "$(dirname "$0")/functions"
 
@@ -61,11 +65,15 @@ done
 # Run autogen.sh on each repository
 for proj in $projects; do
     # autogen.sh is quite verbose, so only print the output in case of failure
-    log_debug "Running autogen.sh for project $proj..."
     (
         cd "$BASEDIR/$proj"
-        export NO_CONFIGURE=1
-        run_and_print_on_failure ./autogen.sh
+        if [ ! -f configure ] || [ ! -f CFVERSION ]; then
+            log_debug "Running autogen.sh for project $proj..."
+            export NO_CONFIGURE=1
+            run_and_print_on_failure ./autogen.sh
+        else
+            log_debug "Skipping autogen.sh in $proj as configure and/or CFVERSION files already exist"
+        fi
     )
 done
 

build-scripts/bootstrap-mission-portal

@@ -0,0 +1,88 @@
+. "$(dirname "$0")"/functions
+. detect-environment
+. compile-options
+. version
+set -e
+
+if [ "$ROLE" = hub ] || [ "$JOB_NAME" = bootstrap-pr ]; then
+    if [ ! -d "$BASEDIR/mission-portal" ] || [ ! -d "$BASEDIR/nova" ]; then
+        fatal "ROLE is $ROLE but $BASEDIR is missing mission-portal and nova repositories"
+    fi
+else
+    log_debug "ROLE ($ROLE) is not hub and JOB_NAME ($JOB_NAME) is not bootstrap-pr so skipping $0"
+    exit 0
+fi
+
+if [ -d "$BASEDIR"/mission-portal/vendor ] && \
+   [ -d "$BASEDIR"/nova/api/http/vendor ] && \
+   [ -d "$BASEDIR"/mission-portal/public/scripts/node_modules ] && \
+   [ -f "$BASEDIR"/mission-portal/public/themese/default/bootstrap/compiled/css/cfengine.less.css ] && \
+   [ -d "$BASEDIR"/mission-portal/ldap/vendor ]; then
+    log_debug "All mission-portal/nova dependencies are installed (by build-in-container.py --tarballs or bootstrap-pr job) so skipping $0"
+    exit 0
+fi
+
+if command -v composer >/dev/null; then
+  COMPOSER=$(command -v composer)
+elif [ -f /usr/local/bin/composer.phar ]; then
+  COMPOSER=/usr/local/bin/composer.phar
+else
+  fatal "Could not find composer command in PATH or at /usr/local/bin/composer.phar"
+  exit 1
+fi
+
+log_debug "Installing javascript npm dependencies..."
+(
+    if test -f "$BASEDIR"/mission-portal/public/scripts/package.json; then
+        cd "$BASEDIR"/mission-portal/public/scripts
+        # display node & npm versions
+        npm --version
+        node --version
+        # install dependencies from npmjs
+        run_and_print_on_failure npm ci --prefix "$BASEDIR"/mission-portal/public/scripts/
+        # build react components
+        run_and_print_on_failure npm run build --prefix "$BASEDIR"/mission-portal/public/scripts/
+        # remove node_modules since the bundles are already built
+        run_and_print_on_failure rm -rf "$BASEDIR"/mission-portal/public/scripts/node_modules
+    fi
+)
+
+log_debug "Installing PHP composer dependencies from mission-portal repository..."
+(
+    if test -f "$BASEDIR"/mission-portal/composer.json; then
+        cd "$BASEDIR"/mission-portal
+        # install PHP dependencies from composer
+        run_and_print_on_failure "$COMPOSER" install --no-dev
+    fi
+)
+
+log_debug "Installing PHP composer dependencies from nova repository..."
+(
+    if test -f "$BASEDIR"/nova/api/http/composer.json; then
+        cd "$BASEDIR"/nova/api/http
+        # install PHP dependencies from composer
+        run_and_print_on_failure "$COMPOSER" install --no-dev --ignore-platform-reqs
+    fi
+)
+
+log_debug "Compiling Mission Portal styles..."
+(
+    if test -f "$BASEDIR"/mission-portal/public/themes/default/bootstrap/cfengine_theme.less; then
+        cd "$BASEDIR"/mission-portal/public/themes/default/bootstrap
+        run_and_print_on_failure npx -p less lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css
+    fi
+)
+
+log_debug "Installing LDAP API PHP composer dependencies..."
+(
+    if test -f "$BASEDIR"/mission-portal/ldap/composer.json; then
+        cd "$BASEDIR"/mission-portal/ldap
+        # install PHP dependencies from composer
+        run_and_print_on_failure php "$COMPOSER" install --no-dev
+    fi
+)
+
+# Composer falls back to git clone when GitHub's anonymous zipball
+# rate limit is hit, leaving non-reproducible .git directories in the
+# vendor tree. Strip them.
+find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} +