Separate retrieve/checkout steps.

Add new buildstep "RETRIEVE" which downloads source code from
remote repositories. This can be invoked singly, using the new
--retrieve <all|packagename> command line option. Clones are
now done with -n (--no-checkout), to save disk space.

Simplify checkout() so that it is only responsible for creating
and updating the src dirs, not cloning git repos. Abe is now much
less tolerant of modifications to the src dir when updates are
done (fixing TCWG-854).

No longer create local_<revision> branches for builds of fixed
revisions.

Update testsuite.

This will allow use of "abe.sh --retrieve all" to create a
snapshots-ref dir directly, without further hacks.

Change-Id: Ie2c6e4044eb159864ff93474cfc47ef9995f4a46

6 files changed, 220 insertions, 100 deletions

diff --git a/abe.sh b/abe.sh
index 9f7f46f1..d8c45f06 100755
--- a/abe.sh
+++ b/abe.sh
@@ -34,6 +34,7 @@ usage()
              [--manifest <manifest_file>]
              [--space <space needed>]
              [--release <release_version_string>]
+             [--retrieve {<package>|all}]
              [--set {arch|cpu|tune}=XXX]
              [--set {buildconfig}=XXX]
              [--set {cflags|ldflags|runtestflags|makeflags}=XXX]
@@ -246,6 +247,17 @@ OPTIONS
 
                     gcc-linaro-4.9-2014.10-1.tar.xz
 
+  --retrieve {<package>|all}
+
+               <package>
+                       This will retrieve the package designated by the
+                       <package> source.conf identifier.
+
+               all
+                       This will retrieve all of the sources for a
+                       complete build as specified by the config/ .conf
+                       files.
+
   --set		{arch|cpu|tune}=XXX
 
 		This overrides the default values used for the configure
@@ -764,6 +776,7 @@ export PATH="${local_builds}/destdir/${build}/bin:$PATH"
 # do_ switches are commands that should be executed after processing all
 # other switches.
 do_dump=
+do_retrieve=
 do_checkout=
 do_makecheck=
 do_excludecheck=
@@ -897,6 +910,14 @@ while test $# -gt 0; do
             release=$2
 	    shift
             ;;
+	--retrieve)
+	    check_directive retrieve $2
+	    # Save and process this after all other elements have been processed.
+	    do_retrieve="$2"
+
+	    # Shift off the 'all' or the package identifier.
+	    shift
+	    ;;
 	--set)
 	    check_directive set "$2"
 
@@ -1250,6 +1271,15 @@ if test ! -z "${do_checkout}" -o ! -z "${do_build}"; then
     fi
 fi
 
+if test ! -z ${do_retrieve}; then
+    if test x"${do_retrieve}" != x"all"; then
+        set_build_component_list "${do_retrieve}"
+    else
+	set_build_component_list "$(get_component_list)"
+    fi
+    set_build_steps retrieve
+fi
+
 if test ! -z ${do_checkout}; then
     if test x"${do_checkout}" != x"all"; then
         set_build_component_list "${do_checkout}"
diff --git a/lib/checkout.sh b/lib/checkout.sh
index 5b3b447d..936cf5a4 100644
--- a/lib/checkout.sh
+++ b/lib/checkout.sh
@@ -42,11 +42,6 @@ checkout_all()
 	if test x"${package}" = x"stage1" -o x"${package}" = x"stage2"; then
 	    package="gcc"
 	fi
-	collect_data ${package}
-	if [ $? -ne 0 ]; then
-	    error "collect_data failed"
-	    return 1
-	fi
 
 	local filespec="$(get_component_filespec ${package})"
 	# don't skip mingw_only components so we get md5sums and/or
@@ -60,25 +55,15 @@ checkout_all()
 		return 1
 	    fi
 	else
-	    fetch ${package}
-	    if test $? -gt 0; then
-		error "Couldn't fetch tarball ${package}"
-		return 1
-	    fi
 	    extract ${package}
 	    if test $? -gt 0; then
-		error "Couldn't extract tarball ${package}"
+		error "Couldn't extract tarball for ${package}"
 		return 1
 	    fi
 	fi
 
     done
 
-    # Reset to the stored value
-    if is_host_mingw && test x"${tarbin}" = xyes; then
-	files="${files} installjammer-1.2.15.tar.gz"
-    fi
-
     notice "Checkout all took ${SECONDS} seconds"
 
     # Set this to no, since all the sources are now checked out
@@ -87,39 +72,25 @@ checkout_all()
     return 0
 }
 
-# various black magic to forcibly update a checkout of a branch in srcdir
-# uses && throughout to avoid verbose error handling
+# merge changes from remote repo
 update_checkout_branch()
 {
     local component="$1"
     local srcdir=
     srcdir="$(get_component_srcdir ${component})" || return 1
-    notice "Updating sources for ${component} in ${srcdir}"
-    dryrun "git -C ${srcdir} checkout -B ${branch} origin/${branch}"
+    local branch=
+    branch="$(get_component_branch ${component})" || return 1
+
+    notice "Updating sources for ${component} in ${srcdir} on ${branch}"
+
+    dryrun "git -C ${srcdir} merge --ff-only origin/${branch}"
     if test $? -gt 0; then
-	error "Can't checkout ${branch}"
-	return 1
+        error "Can't merge changes from ${branch}"
+        return 1
     fi
-    dryrun "git -C ${srcdir} stash --all" &&
-    dryrun "git -C ${srcdir} reset --hard" &&
-    dryrun "git -C ${srcdir} pull" &&
-    # This is required due to the following scenario:  A git
-    # reference dir is populated with a git clone on day X.  On day
-    # Y a developer removes a branch and then replaces the same
-    # branch with a new branch of the same name.  On day Z ABE is
-    # executed against the reference dir copy and the git pull fails
-    # due to error: 'refs/remotes/origin/<branch>' exists; cannot
-    # create 'refs/remotes/origin/<branch>'.  You have to remove the
-    # stale branches before pulling the new ones.
-    dryrun "git -C ${srcdir} remote prune origin" &&
-
-    dryrun "git -C ${srcdir} pull" &&
-    # Update branch directory (which maybe the same as repo
-    # directory)
-    dryrun "git -C ${srcdir} stash --all" &&
-    dryrun "git -C ${srcdir} reset --hard"
 }
 
+# make sure the tag checkout is up to date
 update_checkout_tag()
 {
     local component="$1"
@@ -127,15 +98,7 @@ update_checkout_tag()
     srcdir="$(get_component_srcdir ${component})" || return 1
     local branch=
     branch="$(get_component_branch ${component})" || return 1
-    if git -C ${srcdir} rev-parse -q --verify origin/${branch}; then
-	error "Unexpectedly not tracking origin/${branch}"
-	return 1
-    fi
-    dryrun "git -C ${srcdir} fetch"
-    if test $? -gt 0; then
-	error "Can't reset to ${branch}"
-	return 1
-    fi
+
     local currev="$(git -C ${srcdir} rev-parse HEAD)"
     local tagrev="$(git -C ${srcdir} rev-parse ${branch})"
     if test x${currev} != x${tagrev}; then
@@ -198,52 +161,23 @@ checkout()
 
     case ${protocol} in
 	git*|http*|ssh*)
-	    # If the master branch doesn't exist, clone it. If it exists,
-	    # update the sources.
-	    if test ! -d ${local_snapshots}/${repo}; then
-		local git_reference_opt=
-		if test -d "${git_reference_dir}/${repo}"; then
-		    local git_reference_opt="--reference ${git_reference_dir}/${repo}"
-		fi
-		notice "Cloning $1 in ${local_snapshots}/${repo}"
-		# Note that we are also configuring the clone to fetch gerrit
-		# changes by default.  Since the git reference repos are
-		# generated by this logic, most of the gerrit changes will
-		# already be present in the reference.
-		dryrun "git clone ${git_reference_opt} --config 'remote.origin.fetch=+refs/changes/*:refs/remotes/changes/*' ${repodir} ${local_snapshots}/${repo}"
-		# The above clone fetches only refs/heads/*, so fetch
-		# refs/changes/* by updating the remote.
-		dryrun "git -C ${local_snapshots}/${repo} remote update -p > /dev/null"
-		if test $? -gt 0; then
-		    error "Failed to clone master branch from ${url} to ${local_snapshots}/${repo}"
-		    return 1
-		fi
-	    fi
-
 	    if test ! -d ${srcdir}; then
-		# By definition a git commit resides on a branch.  Therefore specifying a
-		# branch AND a commit is redundant and potentially contradictory.  For this
-		# reason we only consider the commit if both are present.
+		# By definition a git commit resides on a branch.  Therefore
+		# specifying a branch AND a commit is redundant and potentially
+		# contradictory.  For this reason we only consider the commit
+		# if both are present.
 		if test x"${revision}" != x""; then
 		    notice "Checking out revision ${revision} for ${component} in ${srcdir}"
 		    dryrun "${NEWWORKDIR} ${local_snapshots}/${repo} ${srcdir} ${revision}"
 		    if test $? -gt 0; then
-			error "Revision ${revision} likely doesn't exist in git repo ${repo}!"
-			return 1
-		    fi
-		    # git checkout of a commit leaves the head in detached state so we need to
-		    # give the current checkout a name.  Use -B so that it's only created if
-		    # it doesn't exist already.
-		    dryrun "git -C ${srcdir} checkout -B local_${revision}"
-		    if test $? -gt 0; then
-			error "Can't checkout ${revision}"
+			error "Failed to create workdir for ${revision}"
 			return 1
 		    fi
 	        else
 		    notice "Checking out branch ${branch} for ${component} in ${srcdir}"
 		    dryrun "${NEWWORKDIR} ${local_snapshots}/${repo} ${srcdir} ${branch}"
 		    if test $? -gt 0; then
-			error "Branch ${branch} likely doesn't exist in git repo ${repo}!"
+			error "Failed to create workdir for ${branch}"
 			return 1
 		    fi
 		    # If the user mistakenly used ~revision instead of
@@ -266,7 +200,7 @@ checkout()
 		    notice "Building explicit revision for ${component}."
 		    # No need to pull.  A commit is a single moment in time
 		    # and doesn't change.
-		    dryrun "git -C ${srcdir} checkout -B local_${revision} ${revision}"
+		    dryrun "git -C ${srcdir} checkout ${revision}"
 		    if test $? -gt 0; then
 			error "Can't checkout ${revision}"
 			return 1
@@ -308,11 +242,6 @@ checkout()
 	    ;;
     esac
 
-    if test $? -gt 0; then
-	error "Couldn't checkout $1 !"
-	return 1
-    fi
-
     if $new_srcdir; then
 	case "${component}" in
 	    gcc)
diff --git a/lib/common.sh b/lib/common.sh
index 0e5ad747..b883c823 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -32,6 +32,7 @@ set -o pipefail
 . "${topdir}/lib/schroot.sh" || exit 1
 . "${topdir}/lib/component.sh" || exit 1
 . "${topdir}/lib/control.sh" || exit 1
+. "${topdir}/lib/retrieve.sh" || exit 1
 
 # if --dryrun is passed to abe.sh, then commands are echoed instead of
 # of executed.
diff --git a/lib/control.sh b/lib/control.sh
index a02c41fe..3bb59aa9 100644
--- a/lib/control.sh
+++ b/lib/control.sh
@@ -30,6 +30,13 @@ declare -A build_step_required
 build_component_list=""
 check_component_list=""
 
+build_step_required[RETRIEVE]=1
+build_step_RETRIEVE()
+{
+    retrieve_all "$build_component_list"
+}
+
+
 build_step_required[CHECKOUT]=1
 build_step_CHECKOUT()
 {
@@ -102,7 +109,7 @@ perform_build_steps()
     notice "enabled build steps (not in order): ${!build_steps[*]}"
 
     local step
-    for step in CHECKOUT MANIFEST BUILD CHECK INSTALL_SYSROOT HELLO_WORLD TARSRC TARBIN; do
+    for step in RETRIEVE CHECKOUT MANIFEST BUILD CHECK INSTALL_SYSROOT HELLO_WORLD TARSRC TARBIN; do
         if [ ! -z "${build_steps[$step]}" ]; then
 	    # this step is enabled
 	    notice "Performing build step $step"
@@ -129,11 +136,16 @@ perform_build_steps()
 set_build_steps()
 {
     case "$1" in
+	retrieve)
+	    build_steps[RETRIEVE]=1
+	    ;;
 	checkout)
+	    build_steps[RETRIEVE]=1
 	    build_steps[CHECKOUT]=1
 	    build_steps[MANIFEST]=1
 	    ;;
 	build)
+	    build_steps[RETRIEVE]=1
 	    build_steps[CHECKOUT]=1
 	    build_steps[MANIFEST]=1
 	    build_steps[BUILD]=1
diff --git a/lib/retrieve.sh b/lib/retrieve.sh
new file mode 100644
index 00000000..e77ad254
--- /dev/null
+++ b/lib/retrieve.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+# 
+#   Copyright (C) 2013, 2014, 2015, 2016, 2017 Linaro, Inc
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+# 
+
+#
+# This retrieves source code from the remote server(s).
+#
+
+# This is similar to make_all except it _just_ gathers sources trees and does
+# nothing else.
+retrieve_all()
+{
+#    trace "$*"
+
+    local packages="$*"
+
+    notice "retrieve_all called for packages: ${packages}"
+
+    for i in ${packages}; do
+	local package=$i
+	if test x"$i" = x"libc"; then
+	    package="${clibrary}"
+	fi
+	if test x"${package}" = x"stage1" -o x"${package}" = x"stage2"; then
+	    package="gcc"
+	fi
+	collect_data ${package}
+	if [ $? -ne 0 ]; then
+	    error "collect_data failed"
+	    return 1
+	fi
+
+	local filespec="$(get_component_filespec ${package})"
+	# don't skip mingw_only components so we get md5sums and/or
+        # git revisions
+	if test "$(component_is_tar ${package})" = no; then
+ 	    local retrieve_ret=
+	    retrieve ${package}
+	    retrieve_ret=$?
+	    if test ${retrieve_ret} -gt 0; then
+		error "Failed retrieve of ${package}."
+		return 1
+	    fi
+	else
+	    fetch ${package}
+	    if test $? -gt 0; then
+		error "Couldn't retrieve tarball for ${package}"
+		return 1
+	    fi
+	fi
+
+    done
+
+    notice "Retrieve all took ${SECONDS} seconds"
+
+    return 0
+}
+
+# This gets the source tree from a remote host
+# $1 - This should be a service:// qualified URL.  If you just
+#       have a git identifier call get_URL first.
+retrieve()
+{
+#    trace "$*"
+
+    local component="$1"
+
+    # None of the following should be able to fail with the code as it is
+    # written today (and failures are therefore untestable) but propagate
+    # errors anyway, in case that situation changes.
+    local url=
+    url="$(get_component_url ${component})" || return 1
+    local repo=
+    repo="$(get_component_filespec ${component})" || return 1
+    local protocol="$(echo ${url} | cut -d ':' -f 1)"
+    local repodir="${url}/${repo}"
+
+    # FIXME: this satisfies the test suite, but would prefer to validate
+    # ${repodir} without network.
+    dryrun "git ls-remote ${repodir} > /dev/null 2>&1"
+    if test $? -ne 0; then
+        error "proper URL required"
+        return 1
+    fi
+
+    # If the master branch doesn't exist, clone it. If it exists,
+    # update the sources.
+    if test ! -d ${local_snapshots}/${repo}; then
+	local git_reference_opt=
+	if test -d "${git_reference_dir}/${repo}"; then
+	    local git_reference_opt="--reference ${git_reference_dir}/${repo}"
+	fi
+	notice "Cloning $1 in ${local_snapshots}/${repo}"
+	# Note that we are also configuring the clone to fetch gerrit
+	# changes by default.  Since the git reference repos are
+	# generated by this logic, most of the gerrit changes will
+	# already be present in the reference.
+	dryrun "git clone ${git_reference_opt} -n --config 'remote.origin.fetch=+refs/changes/*:refs/remotes/changes/*' ${repodir} ${local_snapshots}/${repo}"
+	if test $? -gt 0; then
+	    error "Failed to clone master branch from ${url} to ${local_snapshots}/${repo}"
+	    return 1
+	fi
+        retrieve_clone_update || return 1
+    else
+        if test x"${supdate}" = xyes; then
+            retrieve_clone_update || return 1
+        fi
+    fi
+
+
+    return 0
+}
+
+retrieve_clone_update()
+{
+    # update local clone with all refs, pruning stale branches
+    dryrun "git -C ${local_snapshots}/${repo} remote update --prune > /dev/null"
+    if test $? -gt 0; then
+        error "Failed to update from ${url} to ${local_snapshots}/${repo}"
+        return 1
+    fi
+}
diff --git a/testsuite/test.sh b/testsuite/test.sh
index f06697c7..6d7f060f 100755
--- a/testsuite/test.sh
+++ b/testsuite/test.sh
@@ -518,10 +518,15 @@ else
 fi
 
 # ----------------------------------------------------------------------------------
-echo "============= checkout () tests ================"
+echo "============= retrieve/checkout () tests ================"
 echo "  Checking out sources into ${local_snapshots}"
 echo "  Please be patient while sources are checked out...."
-echo "================================================"
+echo "========================================================="
+
+retrievecheckout()
+{
+    retrieve "$@" && checkout "$@"
+}
 
 # These can be painfully slow so test small repos.
 
@@ -532,9 +537,9 @@ rm -rf "${local_snapshots}"/*.git*
 testing="http://abe.git@git.linaro.org/git/toolchain/abe.git"
 in="abe"
 if test x"${debug}" = xyes; then
-  out="`cd ${local_snapshots} && checkout ${in}`"
+  out="`cd ${local_snapshots} && retrievecheckout ${in}`"
 else
-  out="`cd ${local_snapshots} && checkout ${in} 2>/dev/null`"
+  out="`cd ${local_snapshots} && retrievecheckout ${in} 2>/dev/null`"
 fi
 if test $? -eq 0; then
   pass "${testing}"
@@ -603,10 +608,17 @@ test_checkout ()
 
     set_component_branch ${package} ${branch}
     set_component_revision ${package} ${revision}
-    set_component_srcdir abe ${local_snapshots}/abe.git~${branch}
+    set_component_srcdir ${package} ${local_snapshots}/${package}.git~${branch}
+
+    retrieve "${package}"
+    ret=$?
+    if test ${ret} -eq 1 -a x"${should}" = x"pass"; then
+        fail "function ${testing} (retrieve)"
+        return 1
+    fi
 
     if test x"${revision}" != x; then
-    set_component_srcdir abe ${local_snapshots}/abe.git_rev_${revision}
+    set_component_srcdir ${package} ${local_snapshots}/${package}.git_rev_${revision}
     fi
 
     if test x"${debug}" = x"yes"; then
@@ -652,7 +664,7 @@ test_checkout ()
 	elif test x"${revision}" = x; then
             branch_test=`(cd ${srcdir} && git branch -a | grep -c "^\* ${branch}$")`
 	else
-            branch_test=`(cd ${srcdir} && git branch -a | grep -c "^\* local_${revision}$")`
+            branch_test=`(cd ${srcdir} && git rev-parse HEAD | grep -c "^${revision}")`
 	fi
     else
 	untested "${testing}"
@@ -717,7 +729,7 @@ test_checkout "${should}" "${testing}" "${package}" "${branch}" "${revision}"
 component_init foo TOOL=foo
 
 testing="checkout: git://testingrepository/foo should fail with 'clone failed' message."
-package="foo.git"
+package="foo"
 branch=''
 revision=''
 should="fail"
@@ -792,7 +804,7 @@ cmp_makeflags=
 
 component_init dejagnu TOOL=dejagnu BRANCH=linaro-local/stable SRCDIR=${local_snapshots}/dejagnu.git~linaro BUILDDIR=${local_builds}/dejagnu.git~linaro FILESPEC=dejagnu.git URL=http://git.linaro.org/git/toolchain
 
-checkout dejagnu
+retrievecheckout dejagnu
 if test $? -eq 0; then
     pass "Checking out Dejagnu for configure test"
 else