lkft: separate the cts/vts, x15 and hikey for aosp master builds

so that we could work around the builder memory problems
https://projects.linaro.org/browse/LSS-985

Change-Id: I5ca30f29d9156c4be371bec05475b7808e374860
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

1 files changed, 333 insertions, 0 deletions

diff --git a/lkft-aosp-master-x15.yaml b/lkft-aosp-master-x15.yaml
new file mode 100644
index 00000000..aa23e75e
--- /dev/null
+++ b/lkft-aosp-master-x15.yaml
@@ -0,0 +1,333 @@
+- job:
+    name: lkft-aosp-master-x15
+    project-type: freestyle
+    defaults: global
+    logrotate:
+        daysToKeep: 60
+        numToKeep: 60
+    properties:
+        - authorization:
+            anonymous:
+                - job-read
+                - job-extended-read
+            linaro:
+                - job-build
+                - job-cancel
+    parameters:
+        - string:
+            name: BUILD_CONFIG_FILENAME
+            default: 'aosp-master-x15'
+        - string:
+            name: QA_SERVER
+            default: 'https://qa-reports.linaro.org'
+        - string:
+            name: QA_SERVER_PROJECT
+            default: 'aosp-master-tracking'
+        - string:
+            name: ANDROID_VERSION
+            default: 'AOSP MASTER'
+        - string:
+            name: TOOLCHAIN
+            default: 'gcc-linaro-7.2.1-2017.11-x86_64_arm-eabi'
+        - string:
+            name: CTS_URL
+            default: 'dummy'
+        - string:
+            name: VTS_URL
+            default: 'dummy'
+    disabled: false
+    node: docker-xenial-aosp
+    display-name: 'LKFT AOSP Master X15 builds'
+    concurrent: true
+    wrappers:
+        - timestamps
+        - timeout:
+            timeout: 1000
+        - credentials-binding:
+            - text:
+                credential-id: QA_REPORTS_TOKEN
+                variable: QA_REPORTS_TOKEN
+            - text:
+                credential-id: ARTIFACTORIAL_TOKEN
+                variable: ARTIFACTORIAL_TOKEN
+    builders:
+        - shell: |
+            #!/bin/bash
+
+            export JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8
+
+            java -version
+
+            sudo apt-get update
+            sudo apt-get install -y gcc-4.9-multilib bison git gperf libxml2-utils python-mako zip time python-requests genisoimage patch mtools python-pip pxz libssl-dev
+
+            wget -q http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre-headless_8u45-b14-1_amd64.deb \
+              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jre_8u45-b14-1_amd64.deb \
+              http://mirrors.kernel.org/ubuntu/pool/universe/o/openjdk-8/openjdk-8-jdk_8u45-b14-1_amd64.deb
+            sudo dpkg -i --force-all *.deb
+
+            mkdir -p ${HOME}/bin ${WORKSPACE}/build/out
+            curl https://storage.googleapis.com/git-repo-downloads/repo > ${HOME}/bin/repo
+            chmod a+x ${HOME}/bin/*
+            export PATH=${HOME}/bin:${PATH}
+
+            DIR_SRV_AOSP_MASTER="/home/buildslave/srv/aosp-master"
+            if [ ! -d "${DIR_SRV_AOSP_MASTER}" ]; then
+              sudo mkdir -p "${DIR_SRV_AOSP_MASTER}"
+              sudo chmod 777 "${DIR_SRV_AOSP_MASTER}"
+            fi
+            cd "${DIR_SRV_AOSP_MASTER}"
+
+            git config --global user.email "ci_notify@linaro.org"
+            git config --global user.name "Linaro CI"
+
+            # Runs as ubuntu
+            set -xe
+            rm -rf .repo/manifests* .repo/local_manifests build-tools jenkins-tools build/out build/android-patchsets build-configs
+            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-android-build-tools.git build-tools
+            git clone --depth 1 https://git.linaro.org/infrastructure/linaro-jenkins-tools.git jenkins-tools
+            git clone --depth 1 http://android-git.linaro.org/git/android-build-configs.git build-configs
+
+            ANDROID_ROOT="${DIR_SRV_AOSP_MASTER}/build"
+            mkdir -p ${ANDROID_ROOT}
+
+            export BUILD_CONFIG_FILENAME=${BUILD_CONFIG_FILENAME:-${JOB_NAME#android-*}}
+
+            # Early test
+            if [ ! -f build-configs/${BUILD_CONFIG_FILENAME} ]; then
+              echo "No config file named ${BUILD_CONFIG_FILENAME} exists"
+              echo "in android-build-configs.git"
+              exit 1
+            fi
+
+            cat << EOF > config.txt
+            BUILD_CONFIG_REPO=http://android-git.linaro.org/git/android-build-configs.git
+            BUILD_CONFIG_BRANCH=master
+            EOF
+            echo config.txt
+            export CONFIG=`base64 -w 0 config.txt`
+
+            # Delete sources before build to save space
+            cd ${ANDROID_ROOT}/
+            rm -rf art/ dalvik/ kernel/ bionic/ developers/ libcore/ sdk/ bootable/ development/ libnativehelper/ system/ build/ device/ test/ build-info/ docs/ packages/ toolchain/ .ccache/ external/ pdk/ tools/ compatibility/ frameworks/ platform_testing/ vendor/ cts/ hardware/ prebuilts/ linaro* clang-src/ hikey-linaro/configs hikey-linaro-4.19
+            rm -fr optee/optee_os
+            # delete this kernel repository as we have patches applied
+            rm -fr hikey-linaro
+            cd -
+
+            # Build Android for X15
+            build-tools/node/build us-east-1.ec2-git-mirror.linaro.org "${CONFIG}"
+            cp -a /home/buildslave/srv/aosp-master/build/out/*.xml /home/buildslave/srv/aosp-master/build/out/*.json ${WORKSPACE}/
+
+            cd ${ANDROID_ROOT}/out
+            [ -n "system.img" ] && X15_SYSTEM_IMG_SHA256SUM=$(sha256sum system.img | cut -d\  -f1)
+            [ -n "userdata.img" ] && X15_USERDATA_IMG_SHA256SUM=$(sha256sum userdata.img | cut -d\  -f1)
+            cd -
+
+            rm -rf build/out/BUILD-INFO.txt
+            wget https://git.linaro.org/ci/job/configs.git/blob_plain/HEAD:/android-lcr/hikey/build-info/aosp-master-template.txt -O build/out/BUILD-INFO.txt
+
+            cd ${ANDROID_ROOT}
+            X15_IMAGE_FILE="
+            MLO
+            u-boot.img
+            boot_fit.img
+            userdata.img
+            system.img
+            vendor.img
+            recovery.img
+            ramdisk.img
+            ramdisk-debug.img
+            "
+            for f in ${X15_IMAGE_FILE}; do
+                if [ -f out/target/product/beagle_x15/${f} ]; then
+                    cp -vf out/target/product/beagle_x15/${f} out/${f}
+                fi
+            done
+            cd -
+
+            cd ${ANDROID_ROOT}
+            # build kernel for x15
+            X15_KERNEL_DIR=${ANDROID_ROOT}/kernel/ti/x15
+            rm -fr ${X15_KERNEL_DIR} && mkdir -p ${X15_KERNEL_DIR}
+            git clone -b android-beagle-x15-4.14 https://android.googlesource.com/kernel/omap ${X15_KERNEL_DIR}
+
+            # compile x15 4.14 kernel
+            cd ${X15_KERNEL_DIR}
+            export PATH=~/srv/toolchain/gcc-linaro-7.2.1-2017.11-x86_64_arm-eabi/bin:$PATH
+            export KERNEL_DESCRIBE_X15_4_14=$(git rev-parse --short HEAD)
+            export KERNEL_VERSION_X15_4_14=$(make kernelversion)
+
+            make ARCH=arm CROSS_COMPILE=arm-eabi- ti_sdk_am57x_android_release_defconfig
+            make ARCH=arm CROSS_COMPILE=arm-eabi- -j$(nproc) zImage dtbs modules
+            cd -
+
+            source build/envsetup.sh
+            lunch beagle_x15-userdebug
+            KERNELDIR=${X15_KERNEL_DIR} make -j$(nproc)
+
+            mv out/boot_fit.img out/boot_fit-4.14-${KERNEL_DESCRIBE_X15_4_14}.img
+            mv out/vendor.img out/vendor-4.14-${KERNEL_DESCRIBE_X15_4_14}.img
+
+            cd out/
+            [ -n "boot_fit-4.14-${KERNEL_DESCRIBE_X15_4_14}.img" ] && X15_BOOT_FIT_4_14_IMG_SHA256SUM=$(sha256sum boot_fit-4.14-${KERNEL_DESCRIBE_X15_4_14}.img | cut -d\  -f1)
+            [ -n "vendor-4.14-${KERNEL_DESCRIBE_X15_4_14}.img" ] && X15_VENDOR_4_14_IMG_SHA256SUM=$(sha256sum vendor-4.14-${KERNEL_DESCRIBE_X15_4_14}.img | cut -d\  -f1)
+            cd -
+
+            #######################################
+            # compile x15 4.19 kernel
+            #######################################
+            cd ${X15_KERNEL_DIR}
+            git checkout android-beagle-x15-4.19 && git pull
+            make mrproper
+
+            export KERNEL_DESCRIBE_X15_4_19=$(git rev-parse --short HEAD)
+            export KERNEL_VERSION_X15_4_19=$(make kernelversion)
+
+            ## revert the change of Revert "usb: gadget: ffs: Fix BUG when userland exits with submitted AIO transfers"
+            git revert --no-edit 8b87305f816cf58b7a4c840b2bedfd37a8c06254
+
+            make ARCH=arm CROSS_COMPILE=arm-eabi- ti_sdk_am57x_android_release_defconfig
+            make ARCH=arm CROSS_COMPILE=arm-eabi- -j$(nproc) zImage dtbs modules
+            cd -
+
+            source build/envsetup.sh
+            lunch beagle_x15-userdebug
+            # only need to re-create the boot_fit.img and vendor.img for 4.19 here
+            KERNELDIR=${X15_KERNEL_DIR}  make -j$(nproc) bootfitimage vendorimage
+
+            X15_IMAGE_FILE="
+            boot_fit.img
+            vendor.img
+            "
+            for f in ${X15_IMAGE_FILE}; do
+                if [ -f out/target/product/beagle_x15/${f} ]; then
+                    cp -vf out/target/product/beagle_x15/${f} out/${f}
+                fi
+            done
+            mv out/boot_fit.img out/boot_fit-4.19-${KERNEL_DESCRIBE_X15_4_19}.img
+            mv out/vendor.img out/vendor-4.19-${KERNEL_DESCRIBE_X15_4_19}.img
+
+            cd out/
+            [ -n "vendor-4.19-${KERNEL_DESCRIBE_X15_4_19}.img" ] && X15_VENDOR_4_19_IMG_SHA256SUM=$(sha256sum vendor-4.19-${KERNEL_DESCRIBE_X15_4_19}.img | cut -d\  -f1)
+            [ -n "boot_fit-4.19-${KERNEL_DESCRIBE_X15_4_19}.img" ] && X15_BOOT_FIT_4_19_IMG_SHA256SUM=$(sha256sum boot_fit-4.19-${KERNEL_DESCRIBE_X15_4_19}.img | cut -d\  -f1)
+            cd -
+
+            # Delete sources after build to save space
+            rm -rf art/ dalvik/ kernel/ bionic/ developers/ libcore/ sdk/ bootable/ development/ libnativehelper/ system/ build/ device/ test/ build-info/ docs/ packages/ toolchain/ .ccache/ external/ pdk/ tools/ compatibility/ frameworks/ platform_testing/ vendor/ cts/ hardware/ prebuilts/ linaro*
+            rm -fr ${X15_KERNEL_DIR}
+
+            # Publish parameters
+            cat << EOF > ${WORKSPACE}/publish_parameters
+            PUB_DEST=android/lkft/lkft-aosp-master-x15/${BUILD_NUMBER}
+            PUB_SRC=${PWD}/out
+            PUB_EXTRA_INC=^[^/]+\.(dtb|dtbo|zip)$|MLO|vmlinux|System.map
+
+            KERNEL_DESCRIBE_X15_4_14=${KERNEL_DESCRIBE_X15_4_14}
+            KERNEL_DESCRIBE_X15_4_19=${KERNEL_DESCRIBE_X15_4_19}
+
+            KERNEL_VERSION_X15_4_14=${KERNEL_VERSION_X15_4_14}
+            KERNEL_VERSION_X15_4_19=${KERNEL_VERSION_X15_4_19}
+            X15_SYSTEM_IMG_SHA256SUM=${X15_SYSTEM_IMG_SHA256SUM}
+            X15_USERDATA_IMG_SHA256SUM=${X15_USERDATA_IMG_SHA256SUM}
+            X15_VENDOR_4_14_IMG_SHA256SUM=${X15_VENDOR_4_14_IMG_SHA256SUM}
+            X15_VENDOR_4_19_IMG_SHA256SUM=${X15_VENDOR_4_19_IMG_SHA256SUM}
+            X15_BOOT_FIT_4_14_IMG_SHA256SUM=${X15_BOOT_FIT_4_14_IMG_SHA256SUM}
+            X15_BOOT_FIT_4_19_IMG_SHA256SUM=${X15_BOOT_FIT_4_19_IMG_SHA256SUM}
+            EOF
+        - inject:
+            properties-file: publish_parameters
+        - linaro-publish-token
+        - shell:
+            !include-raw:
+                - android/linaro-publisher.sh
+        - inject:
+            properties-file: publish_parameters
+        - shell: |
+            # Install ruamel.yaml
+            pip install --user --force-reinstall ruamel.yaml
+            pip install --user --force-reinstall Jinja2
+            export PUB_DEST=android/lkft/lkft-aosp-master-x15/${BUILD_NUMBER}
+            export DEVICE_TYPE=x15
+            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/
+            export DOWNLOAD_URL=http://snapshots.linaro.org/${PUB_DEST}
+            export REFERENCE_BUILD_URL=http://snapshots.linaro.org/${PUB_DEST}
+            export KERNEL_COMMIT=${BUILD_NUMBER}
+            export TOOLCHAIN=${TOOLCHAIN}
+            export KERNEL_REPO=omap
+            export ANDROID_VERSION=aosp-master-$(date --iso-8601)
+            export VTS_VERSION=$(echo $VTS_URL | awk -F"/" '{print$(NF-1)}')
+            export CTS_VERSION=$(echo $CTS_URL | awk -F"/" '{print$(NF-1)}')
+            export QA_BUILD_VERSION=${BUILD_NUMBER}
+
+            rm -rf configs
+            git clone --depth 1 http://git.linaro.org/ci/job/configs.git
+
+            # submit jobs for X15 4.14 kernel build
+            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_X15_4_14}
+            export SRCREV_kernel=${KERNEL_DESCRIBE_X15_4_14}
+            export KERNEL_BRANCH="android-beagle-x15-${KERNEL_VERSION_X15_4_14}-${KERNEL_DESCRIBE_X15_4_14}"
+            # submit x15 jobs to lkf.v.l.o instance
+            export LAVA_SERVER=https://lkft.validation.linaro.org/RPC2/
+
+            # reset the changes done before
+            cd configs && git reset --hard && cd -
+
+            # update templates to use the new image names
+            sed -i "s|boot_fit.img|x15-boot_fit-4.14-${KERNEL_DESCRIBE_X15_4_14}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
+            sed -i "s|vendor.img|x15-vendor-4.14-${KERNEL_DESCRIBE_X15_4_14}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
+
+            export X15_BOOT_FIT_IMG_SHA256SUM=${X15_BOOT_FIT_4_14_IMG_SHA256SUM}
+            export X15_VENDOR_IMG_SHA256SUM=${X15_VENDOR_4_14_IMG_SHA256SUM}
+            python configs/openembedded-lkft/submit_for_testing.py \
+                --device-type ${DEVICE_TYPE} \
+                --build-number ${BUILD_NUMBER} \
+                --lava-server ${LAVA_SERVER} \
+                --qa-server ${QA_SERVER} \
+                --qa-server-team android-lkft \
+                --env-suffix "_4.14" \
+                --qa-server-project ${QA_SERVER_PROJECT} \
+                --git-commit ${QA_BUILD_VERSION} \
+                --testplan-path configs/lkft/lava-job-definitions/x15 \
+                --test-plan template-vts-kernel.yaml \
+                --quiet
+
+            # submit jobs for x15 4.19 kernel build
+            # reset the changes done before
+            cd configs && git reset --hard && cd -
+
+            export KERNEL_DESCRIBE=${KERNEL_DESCRIBE_X15_4_19}
+            export SRCREV_kernel=${KERNEL_DESCRIBE_X15_4_19}
+            export KERNEL_BRANCH="android-beagle-x15-${KERNEL_VERSION_X15_4_19}-${KERNEL_DESCRIBE_X15_4_19}"
+
+            # update templates to use the new image names
+            sed -i "s|boot_fit.img|x15-boot_fit-4.19-${KERNEL_DESCRIBE_X15_4_19}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
+            sed -i "s|vendor.img|x15-vendor-4.19-${KERNEL_DESCRIBE_X15_4_19}.img|g" configs/lkft/lava-job-definitions/${DEVICE_TYPE}/*.yaml
+
+            export X15_BOOT_FIT_IMG_SHA256SUM=${X15_BOOT_FIT_4_19_IMG_SHA256SUM}
+            export X15_VENDOR_IMG_SHA256SUM=${X15_VENDOR_4_19_IMG_SHA256SUM}
+            python configs/openembedded-lkft/submit_for_testing.py \
+                --device-type ${DEVICE_TYPE} \
+                --build-number ${BUILD_NUMBER} \
+                --lava-server ${LAVA_SERVER} \
+                --qa-server ${QA_SERVER} \
+                --qa-server-team android-lkft \
+                --env-suffix "_4.19" \
+                --qa-server-project ${QA_SERVER_PROJECT} \
+                --git-commit ${QA_BUILD_VERSION} \
+                --testplan-path configs/lkft/lava-job-definitions/x15 \
+                --test-plan template-vts-kernel.yaml \
+                --quiet
+
+    publishers:
+        - archive:
+            artifacts: '*.xml,*.json'
+            latest-only: true
+        - logparser:
+            parse-rules: '/var/jenkins_home/userContent/android.parse'
+            unstable-on-warning: false
+            fail-on-error: false
+        - fingerprint:
+            files: 'build/fingerprints/*'
+        - email:
+            recipients: 'tom.gall@linaro.org yongqin.liu@linaro.org'