Clarify and fix handling of md5sums in $git_reference_dir

After recent improvements to checkout code fetch_md5sums started
to treat $git_reference_dir/md5sums as the intended source of md5sums
data.  While copying of [mostly infrastructure/] files from
$git_referece_dir is OK and beneficial, md5sums is not versioned in
the filename, and appearance of new version on $fileserver does not
invalidate the copy in $git_reference_dir.

The patch has a double fix for the problem.  Firstly, it removes code
from fetch_md5sums that handles $git_reference_dir, leaving two options
for md5sums: either be present in the first place, or be fetched via http.

Secondly, it updates scripts/update-snapshots-ref.sh to remove md5sums
after generation of reference snapshots.

Change-Id: Iaef5f9f729aef75f2b5ad0c60cf666fafd5d6f57

2 files changed, 14 insertions, 12 deletions

diff --git a/lib/fetch.sh b/lib/fetch.sh
index 594643bb..02d0139e 100755
--- a/lib/fetch.sh
+++ b/lib/fetch.sh
@@ -23,15 +23,13 @@
 # of md5sums.
 fetch_md5sums()
 {
-    if test "${git_reference_dir:+set}" = "set" -a -e "${git_reference_dir}/md5sums"; then
-	# The user specified that they want to fetch from the reference dir.  This
-	# will always fetch if the version in the reference dir is newer.
-	fetch_reference md5sums
-    else
-	# The fetch_http function will always attempt to fetch the remote file
-	# if the version on the server is newer than the local version.
-	fetch_http md5sums
-    fi
+    # The fetch_http function will always attempt to fetch the remote file
+    # if the version on the server is newer than the local version.
+    # Note, we don't want to fetch md5sums from $git_reference_dir because
+    # it is not versioned in the filename like other gmp-* or linux-* files
+    # are.  $git_reference_dir is a caching mechanism serving fast access to
+    # git/file content, but it doesn't provide any master copies of content.
+    fetch_http md5sums
 
     # If the fetch_*() fails we might have a previous version of md5sums in
     # ${local_snapshots}.  Use that, otherwise we have no choice but to fail.
diff --git a/scripts/update-snapshots-ref.sh b/scripts/update-snapshots-ref.sh
index 31ff4464..0abcc60e 100755
--- a/scripts/update-snapshots-ref.sh
+++ b/scripts/update-snapshots-ref.sh
@@ -64,10 +64,14 @@ generate_snapshots ()
 
 if $generate; then
     generate_snapshots
+
+    # Remove checked-out branch directories
+    rm -rf ${snapshots_dir}-new/*~*
+
+    # Remove md5sums to force ABE to fetch canonical version via http://.
+    rm ${snapshots_dir}-new/md5sums
 fi
 
-# Remove checked-out branch directories
-rm -rf $snapshots_dir-new/*~*
 
 # Cleanup stale branches
 for repo in $snapshots_dir-new/*.git; do
@@ -92,7 +96,7 @@ if true; then
     for M in $todo_machines; do
 	(
 	    rsync -az --delete $snapshots_dir-new/ $M:$snapshots_dir-new/
-	    ssh -fn $M "flock -x $snapshots_dir.lock -c \"rsync -a $snapshots_dir-new/ $snapshots_dir/\""
+	    ssh -fn $M "flock -x $snapshots_dir.lock -c \"rsync -a --delete ${snapshots_dir}-new/ $snapshots_dir/\""
 	) > /tmp/update-snapshots-ref.$$.$M 2>&1 &
 	pids[$M]=$!
     done