lhg-oe-build: switch to docker slave, make closer to rpb-oe jobs

Otherwise mostly cleanups. The visible changes is the directory layout
at snapshots: ${MACHINE}/${BUILD_NUMBER}/${DISTRO} vs previously used
${MACHINE}/${DISTRO}/${BUILD_NUMBER}.

Change-Id: I8bd9d789de9d0abfb436ad6ce5797b88bfcd1925
Signed-off-by: Andrey Konovalov <andrey.konovalov@linaro.org>

2 files changed, 130 insertions, 194 deletions

diff --git a/lhg-oe-build/builders-docker.sh b/lhg-oe-build/builders-docker.sh
deleted file mode 100755
index 15f31d4e..00000000
--- a/lhg-oe-build/builders-docker.sh
+++ /dev/null
@@ -1,171 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# workaround EDK2 is confused by the long path used during the build
-# and truncate files name expected by VfrCompile
-sudo mkdir -p /srv/oe
-sudo chown buildslave:buildslave /srv/oe
-cd /srv/oe
-
-trap cleanup_exit INT TERM EXIT
-
-cleanup_exit()
-{
-    echo "Running cleanup_exit..."
-}
-
-if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update; then
-  echo "INFO: apt update error - try again in a moment"
-  sleep 15
-  sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update || true
-fi
-pkg_list="android-tools-fsutils chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev python-crypto python-requests texinfo vim-tiny whiptail"
-if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; then
-  echo "INFO: apt install error - try again in a moment"
-  sleep 15
-  sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}
-fi
-
-set -ex
-
-mkdir -p ${HOME}/bin
-curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
-chmod a+x ${HOME}/bin/repo
-export PATH=${HOME}/bin:${PATH}
-
-# initialize repo if not done already
-# if ${MANIFEST_BRANCH} is not set or empty set it to default value
-if [ "x${MANIFEST_BRANCH}" = "x" ]; then
-   https://github.com/96boards/oe-rpb-manifest.git
-fi
-if [ ! -e ".repo/manifest.xml" ]; then
-   repo init -u ${MANIFEST_REPO} -b ${MANIFEST_BRANCH}
-
-   # link to shared downloads on persistent disk
-   # our builds config is expecting downloads and sstate-cache, here.
-   # DL_DIR = "${OEROOT}/sources/downloads"
-   # SSTATE_DIR = "${OEROOT}/build/sstate-cache"
-   mkdir -p ${HOME}/srv/oe/downloads ${HOME}/srv/oe/sstate-cache-${DISTRO}
-   mkdir -p build
-   ln -s ${HOME}/srv/oe/downloads
-   ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO} sstate-cache
-fi
-
-repo sync
-cp .repo/manifest.xml source-manifest.xml
-repo manifest -r -o pinned-manifest.xml
-MANIFEST_COMMIT=$(cd .repo/manifests && git rev-parse --short HEAD)
-
-# the setup-environment will create auto.conf and site.conf
-# make sure we get rid of old config.
-# let's remove the previous TMPDIR as well.
-# we want to preserve build/buildhistory though.
-rm -rf conf build/conf build/tmp-*glibc/
-
-# Accept EULA if/when needed
-export EULA_dragonboard410c=1
-export EULA_stih410b2260=1
-source setup-environment build
-
-# Add job BUILD_NUMBER to output files names
-cat << EOF >> conf/auto.conf
-IMAGE_NAME_append = "-${BUILD_NUMBER}"
-KERNEL_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
-MODULE_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
-DT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
-BOOT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
-EOF
-
-# get build stats to make sure that we use sstate properly
-cat << EOF >> conf/auto.conf
-INHERIT += "buildstats buildstats-summary"
-EOF
-
-# add useful debug info
-cat conf/{site,auto}.conf
-
-# if ${IMAGES} is not set or empty, set it to default
-if [ "x${IMAGES}" = "x" ]; then
-    [ "${DISTRO}" = "rpb" ] && IMAGES+=" rpb-desktop-image rpb-desktop-image-lava"
-    [ "${DISTRO}" = "rpb-wayland" ] && IMAGES+=" rpb-weston-image rpb-weston-image-lava"
-    [ "${MACHINE}" = "am57xx-evm" ] && IMAGES="rpb-console-image"
-fi
-
-if [ "${MACHINE}" = "hikey-32" ] ; then
-    bitbake_secondary_image --extra-machine hikey ${IMAGES}
-else
-    bitbake ${IMAGES}
-fi
-DEPLOY_DIR_IMAGE=$(bitbake -e | grep "^DEPLOY_DIR_IMAGE="| cut -d'=' -f2 | tr -d '"')
-
-# Prepare files to publish
-rm -f ${DEPLOY_DIR_IMAGE}/*.txt
-rm -rf ${DEPLOY_DIR_IMAGE}/bootloader
-find ${DEPLOY_DIR_IMAGE} -type l -delete
-mv /srv/oe/{source,pinned}-manifest.xml ${DEPLOY_DIR_IMAGE}
-cat ${DEPLOY_DIR_IMAGE}/pinned-manifest.xml
-
-# FIXME: Sparse images here, until it gets done by OE
-# FIXME: am57xx-evm creates tar.xz rootfs image and
-#        meta-ti u-boot doesn't enable fastboot support
-case "${MACHINE}" in
-  am57xx-evm|juno|stih410-b2260)
-    ;;
-  *)
-    for rootfs in ${DEPLOY_DIR_IMAGE}/*.rootfs.ext4.gz; do
-      gunzip -k ${rootfs}
-      sudo ext2simg -v ${rootfs%.gz} ${rootfs%.ext4.gz}.img
-      rm -f ${rootfs%.gz}
-      gzip -9 ${rootfs%.ext4.gz}.img
-    done
-    ;;
-esac
-
-# Move HiKey's bootloader related files into its own subdir
-[ "${MACHINE}" = "hikey" ] && {
-  mkdir -p ${DEPLOY_DIR_IMAGE}/bootloader
-  (cd ${DEPLOY_DIR_IMAGE} && mv fip.bin hisi-idt.py l-loader.bin nvme.img ptable-linux-*.img bootloader/)
-}
-
-# Create MD5SUMS file
-find ${DEPLOY_DIR_IMAGE} -type f | xargs md5sum > MD5SUMS.txt
-sed -i "s|${DEPLOY_DIR_IMAGE}/||" MD5SUMS.txt
-mv MD5SUMS.txt ${DEPLOY_DIR_IMAGE}
-
-# Build information - this is 96boards specific
-#cat > ${DEPLOY_DIR_IMAGE}/HEADER.textile << EOF
-#
-#h4. Reference Platform Build - CE OpenEmbedded
-#
-#Build description:
-#* Build URL: "$BUILD_URL":$BUILD_URL
-#* Manifest URL: "https://github.com/96boards/oe-rpb-manifest.git":https://github.com/96boards/oe-rpb-manifest.git
-#* Manifest branch: ${MANIFEST_BRANCH}
-#* Manifest commit: "${MANIFEST_COMMIT}":https://github.com/96boards/oe-rpb-manifest/commit/${MANIFEST_COMMIT}
-#EOF
-
-# Need different files for each machine
-BASE_URL=https://builds.96boards.org/snapshots/reference-platform/openembedded/${MANIFEST_BRANCH}/${MACHINE}/${DISTRO}/${BUILD_NUMBER}
-BOOT_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*-${MACHINE}-*-${BUILD_NUMBER}.img" | xargs -r basename)
-ROOTFS_EXT4_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-lava-${MACHINE}-*-${BUILD_NUMBER}.rootfs.ext4.gz" | xargs -r basename)
-ROOTFS_TARXZ_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-lava-${MACHINE}-*-${BUILD_NUMBER}.rootfs.tar.xz" | xargs -r basename)
-KERNEL_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "uImage-*-${MACHINE}-*-${BUILD_NUMBER}.bin" | xargs -r basename)
-case "${MACHINE}" in
-  am57xx-evm|juno)
-    # FIXME: several dtb files case
-    ;;
-  *)
-    DTB_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*-${MACHINE}-*-${BUILD_NUMBER}.dtb" | xargs -r basename)
-    ;;
-esac
-
-cat << EOF > ${WORKSPACE}/post_build_lava_parameters
-DEPLOY_DIR_IMAGE=${DEPLOY_DIR_IMAGE}
-BOOT_URL=${BASE_URL}/${BOOT_IMG}
-ROOTFS_BUILD_URL=${BASE_URL}/${ROOTFS_EXT4_IMG}
-SYSTEM_URL=${BASE_URL}/${ROOTFS_EXT4_IMG}
-KERNEL_URL=${BASE_URL}/${KERNEL_IMG}
-DTB_URL=${BASE_URL}/${DTB_IMG}
-NFSROOTFS_URL=${BASE_URL}/${ROOTFS_TARXZ_IMG}
-EOF
diff --git a/lhg-oe-build/builders.sh b/lhg-oe-build/builders.sh
index 73410775..0fee4128 100644..100755
--- a/lhg-oe-build/builders.sh
+++ b/lhg-oe-build/builders.sh
@@ -1,10 +1,31 @@
 #!/bin/bash
 
+set -e
+
 # workaround EDK2 is confused by the long path used during the build
 # and truncate files name expected by VfrCompile
-DIR_FOR_WORK=${HOME}/`dirname ${JOB_NAME}`
-mkdir -p ${DIR_FOR_WORK}
-cd ${DIR_FOR_WORK}
+sudo mkdir -p /srv/oe
+sudo chown buildslave:buildslave /srv/oe
+cd /srv/oe
+
+trap cleanup_exit INT TERM EXIT
+
+cleanup_exit()
+{
+    echo "Running cleanup_exit..."
+}
+
+if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update; then
+  echo "INFO: apt update error - try again in a moment"
+  sleep 15
+  sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 update || true
+fi
+pkg_list="android-tools-fsutils chrpath cpio diffstat gawk libmagickwand-dev libmath-prime-util-perl libsdl1.2-dev libssl-dev python-crypto python-requests texinfo vim-tiny whiptail"
+if ! sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}; then
+  echo "INFO: apt install error - try again in a moment"
+  sleep 15
+  sudo DEBIAN_FRONTEND=noninteractive apt-get -q=2 install -y ${pkg_list}
+fi
 
 set -ex
 
@@ -13,47 +34,133 @@ curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
 chmod a+x ${HOME}/bin/repo
 export PATH=${HOME}/bin:${PATH}
 
-# initialize repo
-rm -rf .repo bitbake layers
-repo init -u https://github.com/linaro-home/lhg-oe-manifests.git -b ${MANIFEST_BRANCH} -m default.xml
-mkdir -p build
+# initialize repo if not done already
+if [ ! -e ".repo/manifest.xml" ]; then
+   repo init -u ${MANIFEST_REPO} -b ${MANIFEST_BRANCH}
+
+   # link to shared downloads on persistent disk
+   # our builds config is expecting downloads and sstate-cache, here.
+   # DL_DIR = "${OEROOT}/sources/downloads"
+   # SSTATE_DIR = "${OEROOT}/build/sstate-cache"
+   mkdir -p ${HOME}/srv/oe/downloads ${HOME}/srv/oe/sstate-cache-${DISTRO}
+   mkdir -p build
+   ln -s ${HOME}/srv/oe/downloads
+   ln -s ${HOME}/srv/oe/sstate-cache-${DISTRO} sstate-cache
+fi
 
 repo sync
 cp .repo/manifest.xml source-manifest.xml
 repo manifest -r -o pinned-manifest.xml
+MANIFEST_COMMIT=$(cd .repo/manifests && git rev-parse --short HEAD)
 
-# the setup-environment will create local.conf, make sure we get rid
-# of old config. Let's remove the previous TMPDIR as well. We want
-# to preserve build/buildhistory though.
+# the setup-environment will create auto.conf and site.conf
+# make sure we get rid of old config.
+# let's remove the previous TMPDIR as well.
+# we want to preserve build/buildhistory though.
 rm -rf conf build/conf build/tmp-*glibc/
 
 # Accept EULA if/when needed
 export EULA_dragonboard410c=1
+export EULA_stih410b2260=1
 source setup-environment build
 
-# Add extra options to local.conf when needed
-if [ -e "../add2local.conf" ]; then
-    cat ../add2local.conf >> conf/local.conf
+# Add job BUILD_NUMBER to output files names
+cat << EOF >> conf/auto.conf
+IMAGE_NAME_append = "-${BUILD_NUMBER}"
+KERNEL_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
+MODULE_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
+DT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
+BOOT_IMAGE_BASE_NAME_append = "-${BUILD_NUMBER}"
+EOF
+
+# get build stats to make sure that we use sstate properly
+cat << EOF >> conf/auto.conf
+INHERIT += "buildstats buildstats-summary"
+EOF
+
+# add useful debug info
+cat conf/{site,auto}.conf
+
+# if ${IMAGES} is not set or empty, set it to default
+if [ "x${IMAGES}" = "x" ]; then
+    [ "${DISTRO}" = "rpb" ] && IMAGES+=" rpb-desktop-image rpb-desktop-image-lava"
+    [ "${DISTRO}" = "rpb-wayland" ] && IMAGES+=" rpb-weston-image rpb-weston-image-lava"
+    [ "${MACHINE}" = "am57xx-evm" ] && IMAGES="rpb-console-image"
 fi
 
-bitbake ${image_type}
+if [ "${MACHINE}" = "hikey-32" ] ; then
+    time bitbake_secondary_image --extra-machine hikey ${IMAGES}
+else
+    time bitbake ${IMAGES}
+fi
 DEPLOY_DIR_IMAGE=$(bitbake -e | grep "^DEPLOY_DIR_IMAGE="| cut -d'=' -f2 | tr -d '"')
 
-# Prepare files to archive
+# Prepare files to publish
 rm -f ${DEPLOY_DIR_IMAGE}/*.txt
 find ${DEPLOY_DIR_IMAGE} -type l -delete
-mv ${DIR_FOR_WORK}/{source,pinned}-manifest.xml ${DEPLOY_DIR_IMAGE}
+mv /srv/oe/{source,pinned}-manifest.xml ${DEPLOY_DIR_IMAGE}
+cat ${DEPLOY_DIR_IMAGE}/pinned-manifest.xml
+
+# FIXME: Sparse images here, until it gets done by OE
+case "${MACHINE}" in
+  juno|stih410-b2260)
+    ;;
+  *)
+    for rootfs in ${DEPLOY_DIR_IMAGE}/*.rootfs.ext4.gz; do
+      gunzip -k ${rootfs}
+      sudo ext2simg -v ${rootfs%.gz} ${rootfs%.ext4.gz}.img
+      rm -f ${rootfs%.gz}
+      gzip -9 ${rootfs%.ext4.gz}.img
+    done
+    ;;
+esac
+
+# Move HiKey's bootloader related files into its own subdir
+[ "${MACHINE}" = "hikey" ] && {
+    rm -rf ${DEPLOY_DIR_IMAGE}/bootloader
+    mkdir -p ${DEPLOY_DIR_IMAGE}/bootloader
+    (cd ${DEPLOY_DIR_IMAGE} && mv fip.bin hisi-idt.py l-loader.bin nvme.img ptable-linux-*.img bootloader/)
+}
 
 # Create MD5SUMS file
-(cd ${DEPLOY_DIR_IMAGE} && md5sum * > MD5SUMS.txt)
+find ${DEPLOY_DIR_IMAGE} -type f | xargs md5sum > MD5SUMS.txt
+sed -i "s|${DEPLOY_DIR_IMAGE}/||" MD5SUMS.txt
+mv MD5SUMS.txt ${DEPLOY_DIR_IMAGE}
+
+# Build information - this is 96boards specific
+#cat > ${DEPLOY_DIR_IMAGE}/HEADER.textile << EOF
+#
+#h4. Reference Platform Build - CE OpenEmbedded
+#
+#Build description:
+#* Build URL: "$BUILD_URL":$BUILD_URL
+#* Manifest URL: "https://github.com/96boards/oe-rpb-manifest.git":https://github.com/96boards/oe-rpb-manifest.git
+#* Manifest branch: ${MANIFEST_BRANCH}
+#* Manifest commit: "${MANIFEST_COMMIT}":https://github.com/96boards/oe-rpb-manifest/commit/${MANIFEST_COMMIT}
+#EOF
 
-# The archive publisher can't handle files located outside
-# ${WORKSPACE} - create the link before archiving.
-rm -f ${WORKSPACE}/out
-ln -s ${DEPLOY_DIR_IMAGE} ${WORKSPACE}/out
+# Need different files for each machine
+BOOT_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "boot-*-${MACHINE}-*-${BUILD_NUMBER}.img" | xargs -r basename)
+ROOTFS_EXT4_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-lava-${MACHINE}-*-${BUILD_NUMBER}.rootfs.ext4.gz" | xargs -r basename)
+ROOTFS_TARXZ_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "rpb-console-image-lava-${MACHINE}-*-${BUILD_NUMBER}.rootfs.tar.xz" | xargs -r basename)
+KERNEL_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "uImage-*-${MACHINE}-*-${BUILD_NUMBER}.bin" | xargs -r basename)
+case "${MACHINE}" in
+  am57xx-evm|juno)
+    # FIXME: several dtb files case
+    ;;
+  *)
+    DTB_IMG=$(find ${DEPLOY_DIR_IMAGE} -type f -name "*-${MACHINE}-*-${BUILD_NUMBER}.dtb" | xargs -r basename)
+    ;;
+esac
 
-# publishing is done in a separate build step, so $DEPLOY_DIR_IMAGE
-# needs to be passed using "Inject environment variables"
+# Note: the main job script allows to override the default value for
+#       BASE_URL and PUB_DEST, typically used for OE RPB builds
 cat << EOF > ${WORKSPACE}/post_build_lava_parameters
 DEPLOY_DIR_IMAGE=${DEPLOY_DIR_IMAGE}
+BOOT_URL=${BASE_URL}${PUB_DEST}/${BOOT_IMG}
+ROOTFS_BUILD_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_EXT4_IMG}
+SYSTEM_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_EXT4_IMG}
+KERNEL_URL=${BASE_URL}${PUB_DEST}/${KERNEL_IMG}
+DTB_URL=${BASE_URL}${PUB_DEST}/${DTB_IMG}
+NFSROOTFS_URL=${BASE_URL}${PUB_DEST}/${ROOTFS_TARXZ_IMG}
 EOF