path: root/platforms-ci.sh

#!/bin/bash

# include some common stuff, like check_workspace, etc..
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${SCRIPT_DIR}/platforms-common.sh

function trap_apt_failure
{
	echo "ERROR: failed to install mandatory package"
	echo "ERROR: aborting the build"
	exit 1
}

job_output_dir=out

PARALLELISM=`getconf _NPROCESSORS_ONLN`

# UEFI builds use the WORSKPACE variable, so save it and restore it later
JENKINS_WORKSPACE=${WORKSPACE}
if [ "$JENKINS_WORKSPACE" == "" ]; then
	JENKINS_WORKSPACE=`pwd`
	echo "Using $JENKINS_WORKSPACE as JENKINS_WORKSPACE"
fi
	
unset WORKSPACE

case "$1" in
	"release" )
		JOB_TYPE=release

		Y=`date +%-y`
		M=`date +%-m`
		D=`date +%-d`
		# if the release is being made at the start of the month
		# we assume we're making the previous month's release late
		if [ "${D}" -lt "17" ]; then
			# check if we need to wrap to the previous year
			if [ "$M" == "1" ]; then
				M=11
				((Y--))
			else
				((M--))
			fi
		fi
		YYMM=`printf "%02d.%02d" $Y $M`
		echo Building release $YYMM
		PINNED="pinned-"
		;;

	"snapshot" )
		JOB_TYPE=snapshot
		YYMM=master
		PINNED="pinned-"
		;;

	* )
		JOB_TYPE=snapshot
		YYMM=master
		PINNED=
esac

echo "platforms-ci.sh: doing a $PINNED $JOB_TYPE build for branch $YYMM"

# pre-requisites

if [ "$JOB_NAME" != "" ]; then

	trap trap_apt_failure ERR

	sudo apt-get update

	sudo apt-get install -y --force-yes bison
	sudo apt-get install -y --force-yes flex
	sudo apt-get install -y --force-yes g++
	sudo apt-get install -y --force-yes git
	sudo apt-get install -y --force-yes libssl-dev
	sudo apt-get install -y --force-yes python-crypto
	sudo apt-get install -y --force-yes python-wand
	sudo apt-get install -y --force-yes uuid-dev
	sudo apt-get install -y --force-yes cpio

	# Additional dependencies needed for this CI script
	sudo apt-get install -y --force-yes zip

	trap ERR

	if [ "$USE_PRE_INSTALLED_TOOLCHAIN" == "yes" ]; then
		# Toolchain
		echo ls ${HOME}/srv/toolchain/
		ls ${HOME}/srv/toolchain
		# Use pre-installed linaro toolchain
		export PATH="${HOME}/srv/toolchain/arm-tc-15.02/bin:${HOME}/srv/toolchain/arm64-tc-15.02/bin:$PATH"
		echo "Using pre-installed toolchain"
		echo "PATH=$PATH"
	fi

	# REPO
	curl https://storage.googleapis.com/git-repo-downloads/repo > ./repo
	chmod a+x repo
	repo=$PWD/repo
	ls -al
else
	echo "Not installing stuff for a non-Jenkins build"
	repo=repo
fi

gcc_major=6
gcc_minor=2
gcc_rev=1
gcc_rel=2016.11
gcc_dir_ver=${gcc_major}.${gcc_minor}-${gcc_rel}
gcc_file_ver=${gcc_major}.${gcc_minor}.${gcc_rev}-${gcc_rel}
gcc_aarch64=gcc-linaro-${gcc_file_ver}-x86_64_aarch64-linux-gnu.tar.xz
gcc_aarch32=gcc-linaro-${gcc_file_ver}-x86_64_arm-linux-gnueabihf.tar.xz
gcc_url=http://releases.linaro.org/components/toolchain/binaries/${gcc_dir_ver}
aarch64gcc_url=${gcc_url}/aarch64-linux-gnu/$gcc_aarch64
aarch32gcc_url=${gcc_url}/arm-linux-gnueabihf/$gcc_aarch32

function download_toolchain
{
	pushd $JENKINS_WORKSPACE
	if [ ! -e $gcc_aarch64 ]; then
		wget $aarch64gcc_url
	else
		echo "We already have $gcc_aarch64"
	fi
	if [ ! -e $gcc_aarch32 ]; then
		wget $aarch32gcc_url
	else
		echo "We already have $gcc_aarch32"
	fi

	echo toolchains downloaded
	ls -al
	popd
}

function install_toolchain
{
	local gccdir=tools/gcc
	echo installing toolchain into $gccdir
	mkdir -p $gccdir
	pushd $gccdir

	if [ -e $JENKINS_WORKSPACE/$gcc_aarch64 ]; then
		tar xf $JENKINS_WORKSPACE/$gcc_aarch64
	else
		echo "ERROR: failed to download aarch64 gcc $gcc_aarch64"
		exit
	fi
	if [ -e $JENKINS_WORKSPACE/$gcc_aarch32 ]; then
		tar xf $JENKINS_WORKSPACE/$gcc_aarch32
	else
		echo "ERROR: failed to download aarch32 gcc $gcc_aarch32"
		exit
	fi

	echo toolchains installed
	ls -al
	popd
}
function create_release_zip
{
	local variant=$1        # variant (eg. juno, fvp, ...)
	local manifest=$2       # manifest we used to build the code
	local os=$3             # OS (eg. oe, android, ...)
	local boot=$4           # bootloader (uboot or uefi)
	local to=$5

	local workspace=workspace-${manifest}
	local from=$workspace/output/${variant}/${variant}-${os}/${boot}

	echo copying output $from $to

	mkdir -p $to
	case $variant in
		juno | juno32 )
			cp -R $workspace/recovery/* $to
			cp -R $from/* $to/SOFTWARE
			if [[ "$variant" == "juno" ]]; then
				cp $workspace/uefi/edk2/Build/ArmJuno/DEBUG_GCC${gcc_major}/FV/BL33_AP_UEFI.fd $to/SOFTWARE/bl33-uefi.bin
			fi
			if [ "$manifest" != "uefi" ] && [ "$manifest" != "pinned-uefi" ]; then
				cp $workspace/output/${variant}/components/${variant}/uboot.bin $to/SOFTWARE/bl33-uboot.bin
			fi
			;;
		fvp | fvp32 )
			cp $workspace/model-scripts/run_model.sh $to
			cp -R $from/* $to
			if [[ "$variant" == "fvp" ]]; then
				cp $workspace/uefi/edk2/Build/ArmVExpress-FVP-AArch64/DEBUG_GCC${gcc_major}/FV/FVP_AARCH64_EFI.fd $to/bl33-uefi.bin
			fi
			if [ "$manifest" != "uefi" ] && [ "$manifest" != "pinned-uefi" ]; then
				cp $workspace/output/${variant}/components/${variant}/uboot.bin $to/bl33-uboot.bin
			fi
			;;
		tc2 )
			cp -R $workspace/vexpress-firmware/* $to
			cp -R $from/* $to/SOFTWARE
			;;
		* )
			echo "WARNING: variant not recognised"
			echo "         you will get a default set of files in your ZIP"
			echo "         and you may not be expecting that."
			cp -R $from/* $to
			;;
	esac

	pushd ${to}

	# Generate report for this build
	# First decide the report filename - use an absolute path so we can change
	# directories and still write to the correct report file
	if [ "$YYMM" == "master" ]; then
		# use date and time in the marker file
		report=$PWD/${to}-`date +%Y%m%d-%H%M`.txt
	else
		report=$PWD/${to}-${YYMM}.txt
	fi

	# Add some job information at the start
	echo "ARM Platforms Release build"   >  ${report}
	echo "Job name:     ${JOB_NAME}"     >> ${report}
	echo "Build number: ${BUILD_NUMBER}" >> ${report}
	echo "Job type:     ${JOB_TYPE}"     >> ${report}
	echo "Branch:       $YYMM"           >> ${report}
	if [ "$PINNED" == "" ]; then
		echo "Using the un-pinned manifest." >> ${report}
	else
		echo "Using the pinned manifest."    >> ${report}
	fi
	echo "" >> $report

	# Add the ms5sums of the files for this build
	echo "md5sum                            file" >> ${report}
	echo "--------------------------------  -----------------------------------" >> ${report}
	find . -name "*" -type f | grep -v -e ${report} | sort | xargs md5sum >> ${report}

	# Add the commit ids for the repos we built from
	echo "" >> $report
	pushd ../$workspace
	sha_list >> $report
	popd

	# zip up the build
	zip -r ../${job_output_dir}/${to}.zip *
	popd
}
function dobuilds
{
	manifest=$1 # pinned-ack, ack, pinned-latest or latest
	branch=$2

	if [ "$USE_PRE_INSTALLED_TOOLCHAIN" == "" ]; then
		download_toolchain
	fi

	workspace=workspace-$1
	mkdir -p $workspace
	pushd $workspace

	# just in case this is a rebuild, clean the workspace
	clean_workspace

	# remove old code and builds before syncing
	rm -rf *

	# just for debugging purposes
	ls -al

	if [ "$USE_PRE_INSTALLED_TOOLCHAIN" == "" ]; then
		install_toolchain
	fi

	echo Using repo command: ${repo} init -u https://git.linaro.org/landing-teams/working/arm/manifest -b ${branch} -m ${manifest}.xml
	${repo} init -u https://git.linaro.org/landing-teams/working/arm/manifest -b ${branch} -m ${manifest}.xml

	retries=10
	while [ $retries != 0 ]; do
		${repo} sync -j${PARALLELISM}
		if [ $? == 0 ]; then
			break
		else
			((retries--))
			echo repo sync failed, $retries attempts remaining
		fi
	done
	if [ $? != 0 ] && [ $retries == 0 ]; then
		echo "ERROR: repo sync failed."
		echo "ERROR: aborting."
		exit 1
	fi

	echo "SHA1 List of repos in the build"
	sha_list

	# just for debugging purposes
	echo pwd=$PWD
	ls -al

	check_workspace debug

	mkdir -p prebuilts/android/fvp
	wget http://releases.linaro.org/android/reference-lcr/fvp/7.0-16.10/ramdisk.img -O prebuilts/android/fvp/ramdisk.img
	mkdir -p prebuilts/android/juno
	wget http://releases.linaro.org/members/arm/android/juno/18.01/ramdisk.img -O prebuilts/android/juno/ramdisk.img
	mkdir -p prebuilts/android/tc2
	wget http://releases.linaro.org/android/reference-lcr/vexpress/6.0-15.11/ramdisk.img -O prebuilts/android/tc2/ramdisk.img

	if [ "$manifest" == "uefi" ] || [ "$manifest" == "pinned-uefi" ]; then
		echo "Doing a UEFI only build"
		extra_params="-f uefi"
	else
		extra_params=""
	fi

        if [ "$manifest" == "latest" ] || [ "$manifest" == "pinned-latest" ]; then
                echo "Doing build only for Busybox and OE Filesystems for all platforms"
                ./build-scripts/build-all.sh -f busybox build
		./build-scripts/build-all.sh -f busybox package
                ./build-scripts/build-all.sh -f oe build
                ./build-scripts/build-all.sh -f oe package
        else
	        ./build-scripts/build-all.sh ${extra_params} build
		./build-scripts/build-all.sh ${extra_params} package
        fi
	popd
}

echo "aarch32 cross compiler:"
which arm-linux-gnueabihf-gcc && arm-linux-gnueabihf-gcc -v

echo "aarch64 cross compiler:"
which aarch64-linux-gnu-gcc && aarch64-linux-gnu-gcc -v

# build the two kernel variants
dobuilds ${PINNED}ack    $YYMM
dobuilds ${PINNED}latest $YYMM
dobuilds ${PINNED}uefi   $YYMM

# Create the directory where we'll place all the stuff we expect to appear on snapshots.linaro.org
mkdir ${job_output_dir}

# Create the recovery directories for each variant
create_release_zip juno  ${PINNED}latest busybox  uboot  juno-latest-busybox-uboot
create_release_zip juno  ${PINNED}latest oe       uboot  juno-latest-oe-uboot
create_release_zip juno  ${PINNED}ack    android  uboot  juno-ack-android-uboot
create_release_zip juno  ${PINNED}uefi   uefi     uefi   juno-uefi
create_release_zip fvp   ${PINNED}latest busybox  uboot  fvp-latest-busybox-uboot
create_release_zip fvp   ${PINNED}ack    busybox  uboot  fvp-ack-busybox-uboot
create_release_zip fvp   ${PINNED}latest oe       uboot  fvp-latest-oe-uboot
create_release_zip fvp   ${PINNED}ack    android  uboot  fvp-ack-android-uboot
create_release_zip fvp   ${PINNED}uefi   uefi     uefi   fvp-uefi
create_release_zip fvp32 ${PINNED}latest busybox  uboot  fvp32-latest-busybox-uboot
create_release_zip fvp32 ${PINNED}ack    busybox  uboot  fvp32-ack-busybox-uboot
create_release_zip fvp32 ${PINNED}latest oe       uboot  fvp32-latest-oe-uboot
#create_release_zip fvp32  ${PINNED}lsk    android  uboot  fvp32-lsk-android-uboot
#create_release_zip fvp32  ${PINNED}uefi   uefi     uefi   fvp32-uefi
create_release_zip juno32 ${PINNED}latest busybox  uboot  juno32-latest-busybox-uboot
create_release_zip juno32 ${PINNED}ack    busybox  uboot  juno32-ack-busybox-uboot
create_release_zip juno32 ${PINNED}latest oe       uboot  juno32-latest-oe-uboot
#create_release_zip juno32  ${PINNED}lsk    android  uboot  juno32-lsk-android-uboot
#create_release_zip juno32  ${PINNED}uefi   uefi     uefi   juno32-uefi

if [ "$JOB_TYPE" != "release" ]; then
	# only generate TC2 zips for the non-release job
	create_release_zip tc2  ${PINNED}latest busybox  uboot    tc2-latest-busybox-uboot
	create_release_zip tc2  ${PINNED}latest oe       uboot    tc2-latest-oe-uboot
	create_release_zip tc2  ${PINNED}ack    android  uboot    tc2-ack-android-uboot
	create_release_zip tc2  ${PINNED}uefi   uefi     uefi     tc2-uefi
else
	dbgfile=$PWD/${job_output_dir}/fvp-ack-android-debug.tar.xz
	echo "Generating FVP Android debug archive: ${dbgfile}"
	pushd workspace-${PINNED}ack/linux/out/fvp/android/
	tar cJf ${dbgfile} System.map vmlinux
	popd

	dbgfile=$PWD/${job_output_dir}/fvp-latest-oe-debug.tar.xz
	echo "Generating FVP OE debug archive: ${dbgfile}"
	pushd workspace-${PINNED}latest/linux/out/fvp/mobile_oe/
	tar cJf ${dbgfile} System.map vmlinux
	popd
fi

echo PWD: $PWD
echo "ls -al"
ls -al

pushd ${job_output_dir}
md5sum * > MD5SUMS

ls -al
popd

# restore the WORKSPACE variable now we're finished building
unset WORKSPACE
export WORKSPACE=${JENKINS_WORKSPACE}