path: root/platforms-ci.sh

#!/bin/bash

# include some common stuff, like check_workspace, etc..
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${SCRIPT_DIR}/platforms-common.sh

VALID_PLATFORM_LIST_ACK=("fvp" "fvp32")
VALID_PLATFORM_LIST_LATEST=("fvp" "fvp32" "juno" "juno32")
VALID_PLATFORM_LIST_UEFI=("fvp" "juno")
VALID_PLATFORM_LIST_SGI=("sgi575")
VALID_PLATFORM_LIST_SGM=("sgm775")

function trap_apt_failure
{
	echo "ERROR: failed to install mandatory package"
	echo "ERROR: aborting the build"
	exit 1
}

job_output_dir=out

PARALLELISM=`getconf _NPROCESSORS_ONLN`

# UEFI builds use the WORSKPACE variable, so save it and restore it later
JENKINS_WORKSPACE=${WORKSPACE}
if [ "$JENKINS_WORKSPACE" == "" ]; then
	JENKINS_WORKSPACE=`pwd`
	echo "Using $JENKINS_WORKSPACE as JENKINS_WORKSPACE"
fi

unset WORKSPACE

case "$1" in
	"release" )
		JOB_TYPE=release

		Y=`date +%-y`
		M=`date +%-m`
		D=`date +%-d`
		# if the release is being made at the start of the month
		# we assume we're making the previous month's release late
		if [ "${D}" -lt "17" ]; then
			# check if we need to wrap to the previous year
			if [ "$M" == "1" ]; then
				M=11
				((Y--))
			else
				((M--))
			fi
		fi
		YYMM=`printf "%02d.%02d" $Y $M`
		echo Building release $YYMM
		PINNED="pinned-"
		;;

	"snapshot" )
		JOB_TYPE=snapshot
		YYMM=master
		PINNED="pinned-"
		;;

	* )
		JOB_TYPE=snapshot
		YYMM=master
		PINNED=
esac

echo "platforms-ci.sh: doing a $PINNED $JOB_TYPE build for branch $YYMM"

# pre-requisites

if [ "$JOB_NAME" != "" ]; then

	trap trap_apt_failure ERR

	sudo apt-get update

	sudo apt-get install -y --force-yes bison
	sudo apt-get install -y --force-yes flex
	sudo apt-get install -y --force-yes g++
	sudo apt-get install -y --force-yes git
	sudo apt-get install -y --force-yes libssl-dev
	sudo apt-get install -y --force-yes python-crypto
	sudo apt-get install -y --force-yes python-wand
	sudo apt-get install -y --force-yes uuid-dev
	sudo apt-get install -y --force-yes cpio

	# Additional dependencies needed for this CI script
	sudo apt-get install -y --force-yes zip

	trap ERR

	if [ "$USE_PRE_INSTALLED_TOOLCHAIN" == "yes" ]; then
		# Toolchain
		echo ls ${HOME}/srv/toolchain/
		ls ${HOME}/srv/toolchain
		# Use pre-installed linaro toolchain
		export PATH="${HOME}/srv/toolchain/arm-tc-15.02/bin:${HOME}/srv/toolchain/arm64-tc-15.02/bin:$PATH"
		echo "Using pre-installed toolchain"
		echo "PATH=$PATH"
	fi

	# REPO
	curl https://storage.googleapis.com/git-repo-downloads/repo > ./repo
	chmod a+x repo
	repo=$PWD/repo
	ls -al
else
	echo "Not installing stuff for a non-Jenkins build"
	repo=repo
fi

gcc_major=6
gcc_minor=2
gcc_rev=1
gcc_rel=2016.11
gcc_dir_ver=${gcc_major}.${gcc_minor}-${gcc_rel}
gcc_file_ver=${gcc_major}.${gcc_minor}.${gcc_rev}-${gcc_rel}
gcc_aarch64=gcc-linaro-${gcc_file_ver}-x86_64_aarch64-linux-gnu.tar.xz
gcc_aarch32=gcc-linaro-${gcc_file_ver}-x86_64_arm-linux-gnueabihf.tar.xz
gcc_url=http://releases.linaro.org/components/toolchain/binaries/${gcc_dir_ver}
aarch64gcc_url=${gcc_url}/aarch64-linux-gnu/$gcc_aarch64
aarch32gcc_url=${gcc_url}/arm-linux-gnueabihf/$gcc_aarch32
gcc_arm_none=https://launchpad.net/gcc-arm-embedded/5.0/5-2016-q3-update/+download/gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2
gcc_arm_none_toolchain=gcc-arm-none-eabi-5_4-2016q3-20160926-linux.tar.bz2


function download_toolchain
{
	pushd $JENKINS_WORKSPACE
	if [ ! -e $gcc_aarch64 ]; then
		wget $aarch64gcc_url
	else
		echo "We already have $gcc_aarch64"
	fi
	if [ ! -e $gcc_aarch32 ]; then
		wget $aarch32gcc_url
	else
		echo "We already have $gcc_aarch32"
	fi
	if [ ! -e $gcc_arm_none_toolchain ]; then
		wget $gcc_arm_none
	else
		echo "We already have $gcc_arm_none_toolchain"
	fi

	echo toolchains downloaded
	ls -al
	popd
}

function install_toolchain
{
	local gccdir=tools/gcc
	echo installing toolchain into $gccdir
	mkdir -p $gccdir
	pushd $gccdir

	if [ -e $JENKINS_WORKSPACE/$gcc_aarch64 ]; then
		tar xf $JENKINS_WORKSPACE/$gcc_aarch64
	else
		echo "ERROR: failed to download aarch64 gcc $gcc_aarch64"
		exit
	fi
	if [ -e $JENKINS_WORKSPACE/$gcc_aarch32 ]; then
		tar xf $JENKINS_WORKSPACE/$gcc_aarch32
	else
		echo "ERROR: failed to download aarch32 gcc $gcc_aarch32"
		exit
	fi
	if [ -e $JENKINS_WORKSPACE/$gcc_arm_none_toolchain ]; then
		tar jxf $JENKINS_WORKSPACE/$gcc_arm_none_toolchain
	else
		echo "ERROR: failed to download $gcc_arm_none"
		exit
	fi

	echo toolchains installed
	ls -al
	popd
}

function create_release_zip
{
	local variant=$1        # variant (eg. juno, fvp, ...)
	local manifest=$2       # manifest we used to build the code
	local os=$3             # OS (eg. oe, android, ...)
	local boot=$4           # bootloader (uboot or uefi)
	local to=$5

	local workspace=workspace-${manifest}

        if [ "$variant" == "sgm775" ] || [ "$variant" == "sgi575" ]; then
                local from=$workspace/output/${variant}
        else
                local from=$workspace/output/${variant}/${variant}-${os}/${boot}
        fi

	echo copying output $from $to

	mkdir -p $to
	case $variant in
		juno | juno32 )
			cp -R $workspace/recovery/* $to
			cp -R $from/* $to/SOFTWARE
			if [[ "$variant" == "juno" ]]; then
				if [ -f $workspace/uefi/edk2/Build/ArmJuno/DEBUG_GCC${gcc_major}/FV/BL33_AP_UEFI.fd ]; then
					echo "Copying $workspace/uefi/edk2/Build/ArmJuno/DEBUG_GCC${gcc_major}/FV/BL33_AP_UEFI.fd to $to/SOFTWARE/bl33-uefi.bin"
					cp $workspace/uefi/edk2/Build/ArmJuno/DEBUG_GCC${gcc_major}/FV/BL33_AP_UEFI.fd $to/SOFTWARE/bl33-uefi.bin
				else
					echo "Copying $workspace/uefi/edk2/Build/ArmJuno/DEBUG_GCC5/FV/BL33_AP_UEFI.fd to $to/SOFTWARE/bl33-uefi.bin"
					cp $workspace/uefi/edk2/Build/ArmJuno/DEBUG_GCC5/FV/BL33_AP_UEFI.fd $to/SOFTWARE/bl33-uefi.bin
				fi
			fi
			if [ "$manifest" != "uefi" ] && [ "$manifest" != "pinned-uefi" ]; then
				cp $workspace/output/${variant}/components/juno/uboot.bin $to/SOFTWARE/bl33-uboot.bin
			fi
			;;
		fvp | fvp32 )
			cp $workspace/model-scripts/fvp/run_model.sh $to
			cp -R $from/* $to
			if [[ "$variant" == "fvp" ]]; then
				if [ -f $workspace/uefi/edk2/Build/ArmVExpress-FVP-AArch64/DEBUG_GCC${gcc_major}/FV/FVP_AARCH64_EFI.fd ]; then
					echo "Copying $workspace/uefi/edk2/Build/ArmVExpress-FVP-AArch64/DEBUG_GCC${gcc_major}/FV/FVP_AARCH64_EFI.fd to $to/bl33-uefi.bin"
					cp $workspace/uefi/edk2/Build/ArmVExpress-FVP-AArch64/DEBUG_GCC${gcc_major}/FV/FVP_AARCH64_EFI.fd $to/bl33-uefi.bin
				else
					echo "Copying $workspace/uefi/edk2/Build/ArmVExpress-FVP-AArch64/DEBUG_GCC5/FV/FVP_AARCH64_EFI.fd to $to/bl33-uefi.bin"
					cp $workspace/uefi/edk2/Build/ArmVExpress-FVP-AArch64/DEBUG_GCC5/FV/FVP_AARCH64_EFI.fd $to/bl33-uefi.bin
				fi
				if [ "$os" == "android" ]; then
					echo "Downloading http://releases.linaro.org/android/reference-lcr/fvp/latest/fvp.img.bz2"
					wget http://releases.linaro.org/android/reference-lcr/fvp/latest/fvp.img.bz2
					echo "Extracting fvp.img.bz2"
					bunzip2 -vf fvp.img.bz2
					echo "Copying fvp.img to $to"
					cp fvp.img $to
				fi
			fi
			if [ "$manifest" != "uefi" ] && [ "$manifest" != "pinned-uefi" ]; then
				cp $workspace/output/${variant}/components/fvp/uboot.bin $to/bl33-uboot.bin
			fi
			;;
		tc2 )
			cp -R $workspace/vexpress-firmware/* $to
			cp -R $from/* $to/SOFTWARE
			;;
		sgm775 )
			cp $workspace/model-scripts/sgm775/run_model.sh $to
			cp $workspace/model-scripts/sgm775/settings_generator.sh $to
			sed -i 's#ROOTDIR="../../output/sgm775"#ROOTDIR="."#' $to/run_model.sh
			cp -R $from/* $to
			;;
		sgi575 )
			cp $workspace/model-scripts/sgi/platforms/sgi575/run_model.sh $to
			sed -i 's#ROOTDIR="../../../../output/sgi575"#ROOTDIR="."#' $to/run_model.sh
			cp $workspace/model-scripts/sgi/sgi_common_util.sh $to
			sed -i 's#source ../../sgi_common_util.sh#source sgi_common_util.sh#' $to/run_model.sh
			cp -aR $from/* $to
			;;
		* )
			echo "WARNING: variant not recognised"
			echo "         you will get a default set of files in your ZIP"
			echo "         and you may not be expecting that."
			cp -R $from/* $to
			;;
	esac

	pushd ${to}

	# Generate report for this build
	# First decide the report filename - use an absolute path so we can change
	# directories and still write to the correct report file
	if [ "$YYMM" == "master" ]; then
		# use date and time in the marker file
		report=$PWD/${to}-`date +%Y%m%d-%H%M`.txt
	else
		report=$PWD/${to}-${YYMM}.txt
	fi

	# Add some job information at the start
	echo "ARM Platforms Release build"   >  ${report}
	echo "Job name:     ${JOB_NAME}"     >> ${report}
	echo "Build number: ${BUILD_NUMBER}" >> ${report}
	echo "Job type:     ${JOB_TYPE}"     >> ${report}
	echo "Branch:       $YYMM"           >> ${report}
	if [ "$PINNED" == "" ]; then
		echo "Using the un-pinned manifest." >> ${report}
	else
		echo "Using the pinned manifest."    >> ${report}
	fi
	echo "" >> $report

	# Add the ms5sums of the files for this build
	echo "md5sum                            file" >> ${report}
	echo "--------------------------------  -----------------------------------" >> ${report}
	find . -name "*" -type f | grep -v -e ${report} | sort | xargs md5sum >> ${report}

	# Add the commit ids for the repos we built from
	echo "" >> $report
	pushd ../$workspace
	sha_list >> $report
	popd

	# zip up the build
	zip --symlinks -r ../${job_output_dir}/${to}.zip *
	popd
}

function dobuilds
{
	manifest=$1 # pinned-ack, ack, pinned-latest or latest
	branch=$2

	if [ "$USE_PRE_INSTALLED_TOOLCHAIN" == "" ]; then
		download_toolchain
	fi

	workspace=workspace-$1
	mkdir -p $workspace
	pushd $workspace

	# just in case this is a rebuild, clean the workspace
	clean_workspace

	# remove old code and builds before syncing
	rm -rf *

	# just for debugging purposes
	ls -al

	if [ "$USE_PRE_INSTALLED_TOOLCHAIN" == "" ]; then
		install_toolchain
	fi

	echo Using repo command: ${repo} init -u https://git.linaro.org/landing-teams/working/arm/manifest -b ${branch} -m ${manifest}.xml
	${repo} init -u https://git.linaro.org/landing-teams/working/arm/manifest -b ${branch} -m ${manifest}.xml

	retries=10
	while [ $retries != 0 ]; do
		${repo} sync -j${PARALLELISM} --fetch-submodules --force-sync
		if [ $? == 0 ]; then
			break
		else
			((retries--))
			echo repo sync failed, $retries attempts remaining
		fi
	done
	if [ $? != 0 ] && [ $retries == 0 ]; then
		echo "ERROR: repo sync failed."
		echo "ERROR: aborting."
		exit 1
	fi

	echo "SHA1 List of repos in the build"
	sha_list

	# just for debugging purposes
	echo pwd=$PWD
	ls -al

	check_workspace debug

	mkdir -p prebuilts/android/fvp
	wget http://releases.linaro.org/android/reference-lcr/fvp/latest/ramdisk.img -O prebuilts/android/fvp/ramdisk.img
	mkdir -p prebuilts/android/juno
	wget http://releases.linaro.org/members/arm/android/juno/19.01/ramdisk.img -O prebuilts/android/juno/ramdisk.img
	mkdir -p prebuilts/android/tc2
	wget http://releases.linaro.org/android/reference-lcr/vexpress/6.0-15.11/ramdisk.img -O prebuilts/android/tc2/ramdisk.img

	if [ "$manifest" == "uefi" ] || [ "$manifest" == "pinned-uefi" ]; then
		echo "Doing a UEFI only build"
		for platform in "${VALID_PLATFORM_LIST_UEFI[@]}"
		do
			./build-scripts/build-all.sh -p ${platform} -f uefi all
		done
	elif [ "$manifest" == "latest" ] || [ "$manifest" == "pinned-latest" ]; then
		echo "Doing build only for Busybox and OE Filesystems"
		for platform in "${VALID_PLATFORM_LIST_LATEST[@]}"
		do
			./build-scripts/build-all.sh -p ${platform} -f busybox all
			./build-scripts/build-all.sh -p ${platform} -f oe all
		done
	elif [ "$manifest" == "sgi575" ] || [ "$manifest" == "pinned-sgi575" ]; then
		echo "Doing build only for SGI Busybox Filesystem"
		for platform in "${VALID_PLATFORM_LIST_SGI[@]}"
		do
			./build-scripts/sgi/build-test-busybox.sh -p ${platform} all
		done
	elif [ "$manifest" == "sgm775" ]; then
		echo "Doing build only for SGM Busybox Filesystem"
		for platform in "${VALID_PLATFORM_LIST_SGM[@]}"
		do
			./build-scripts/build-all.sh -p ${platform} -f busybox all
		done
	elif [ "$manifest" == "sgm775-android" ]; then
		echo "Doing build only for SGM Android Filesystem"
		for platform in "${VALID_PLATFORM_LIST_SGM[@]}"
		do
			./build-scripts/build-all.sh -p ${platform} -f android all
		done
	elif [ "$manifest" == "pinned-juno" ]; then
		echo "Doing build only for juno Android and juno32 busybox Filesystem"
		./build-scripts/build-all.sh -p juno -f android all
		./build-scripts/build-all.sh -p juno32 -f busybox all
	elif [ "$manifest" == "pinned-ack" ]; then
		echo "Doing build only for fvp Android and fvp & fvp32 busybox Filesystem"
		./build-scripts/build-all.sh -p fvp -f android all
		./build-scripts/build-all.sh -p fvp -f busybox all
		./build-scripts/build-all.sh -p fvp32 -f busybox all
	fi
	popd
}

echo "aarch32 cross compiler:"
which arm-linux-gnueabihf-gcc && arm-linux-gnueabihf-gcc -v

echo "aarch64 cross compiler:"
which aarch64-linux-gnu-gcc && aarch64-linux-gnu-gcc -v

# Create the directory where we'll place all the stuff we expect to appear on snapshots.linaro.org
mkdir ${job_output_dir}

# Build the variants and Create the release bundle for each variant
dobuilds ${PINNED}ack    $YYMM
create_release_zip fvp    ${PINNED}ack    android  uboot  fvp-ack-android-uboot
create_release_zip fvp    ${PINNED}ack    busybox  uboot  fvp-ack-busybox-uboot
create_release_zip fvp32  ${PINNED}ack    busybox  uboot  fvp32-ack-busybox-uboot

dobuilds ${PINNED}latest $YYMM
create_release_zip fvp    ${PINNED}latest busybox  uboot  fvp-latest-busybox-uboot
create_release_zip fvp    ${PINNED}latest oe       uboot  fvp-latest-oe-uboot
create_release_zip fvp32  ${PINNED}latest busybox  uboot  fvp32-latest-busybox-uboot
create_release_zip fvp32  ${PINNED}latest oe       uboot  fvp32-latest-oe-uboot
create_release_zip juno   ${PINNED}latest busybox  uboot  juno-latest-busybox-uboot
create_release_zip juno   ${PINNED}latest oe       uboot  juno-latest-oe-uboot
create_release_zip juno32 ${PINNED}latest busybox  uboot  juno32-latest-busybox-uboot
create_release_zip juno32 ${PINNED}latest oe       uboot  juno32-latest-oe-uboot

dobuilds ${PINNED}uefi   $YYMM
create_release_zip fvp    ${PINNED}uefi   uefi     uefi   fvp-uefi
create_release_zip juno   ${PINNED}uefi   uefi     uefi   juno-uefi
#create_release_zip fvp32  ${PINNED}uefi   uefi     uefi   fvp32-uefi
#create_release_zip juno32  ${PINNED}uefi  uefi     uefi   juno32-uefi

dobuilds ${PINNED}juno   $YYMM
create_release_zip juno   ${PINNED}juno   android  uboot  juno-ack-android-uboot
create_release_zip juno32 ${PINNED}juno   busybox  uboot  juno32-ack-busybox-uboot

dobuilds ${PINNED}sgi575          $YYMM
create_release_zip sgi575 ${PINNED}sgi575          busybox  uefi   sgi575-latest-busybox-uefi

dobuilds sgm775          $YYMM
create_release_zip sgm775 sgm775          busybox  uboot  sgm775-ack-busybox-uboot

dobuilds sgm775-android  $YYMM
create_release_zip sgm775 sgm775-android  android  uboot  sgm775-ack-android-uboot

echo PWD: $PWD
echo "ls -al"
ls -al

pushd ${job_output_dir}
md5sum * > MD5SUMS

ls -al
popd

# restore the WORKSPACE variable now we're finished building
unset WORKSPACE
export WORKSPACE=${JENKINS_WORKSPACE}