[merge] Tools for working with universal seed ("uniseed").

2 files changed, 146 insertions, 0 deletions

diff --git a/utils/seeded-builds/manifest-union b/utils/seeded-builds/manifest-union
new file mode 100755
index 0000000..c27244c
--- /dev/null
+++ b/utils/seeded-builds/manifest-union
@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+"""
+Produce set-theorhetic union of several Android manifests. Each project
+appearing in any manifest gets output once, and at most once, to the
+resulting manifest. Differences in checkout path/branch are ignored -
+values from a random occurance are output in result. The output is
+mostly intended for usage with "repo init --mirror", to produce mirror
+which contains all repositories used by a number of manifests.
+"""
+import sys
+import optparse
+from xml.dom import minidom
+
+
+class Manifest:
+
+    def __init__(self):
+        self.remote_map = {}
+        self.project_map = {}
+        self.default_node = None
+
+    @staticmethod
+    def compare_attr_wise(node1, node2):
+        if node1.attributes.length != node2.attributes.length:
+            return False
+        for i in xrange(node1.attributes.length):
+            a1 = node1.attributes.item(i)
+            if a1.value != node2.attributes[a1.name].value:
+                return False
+        return True
+
+    def add_default(self, default_node):
+        if self.default_node:
+            if not self.compare_attr_wise(default_node, self.default_node):
+                raise ValueError("default collision: %s vs %s" % (default_node.toxml(), self.default_node.toxml()))
+            return
+        self.default_node = default_node
+
+    def add_remote(self, remote_node):
+        name = remote_node.getAttribute("name")
+        if name in self.remote_map:
+            if not self.compare_attr_wise(remote_node, self.remote_map[name]):
+                raise ValueError("remote collision: %s vs %s" % (remote_node.toxml(), self.remote_map[name].toxml()))
+            return False
+        self.remote_map[name] = remote_node
+        return True
+
+    def add_project(self, project_node):
+        remote = project_node.getAttribute("remote")
+        name = project_node.getAttribute("name")
+        revision = project_node.getAttribute("revision")
+        path = project_node.getAttribute("path")
+#        key = (remote, name, revision, path)
+        key = (remote, name)
+        if key in self.project_map:
+            return False
+        self.project_map[key] = project_node
+        return True
+
+    def get_default(self):
+        return self.default_node
+
+    def get_remotes(self):
+        return self.remote_map.values()
+
+    def get_projects(self):
+        return self.project_map.values()
+
+
+def union(manifest_files):
+    manifest = Manifest()
+    first = True
+    for f in manifest_files:
+        print >>sys.stderr, "Processing: %s" % f
+        dom = minidom.parse(f)
+        default = dom.getElementsByTagName("default")
+        assert len(default) == 1
+        manifest.add_default(default[0])
+        for r in dom.getElementsByTagName("remote"):
+            manifest.add_remote(r)
+        for p in dom.getElementsByTagName("project"):
+            added = manifest.add_project(p)
+            if not first and added:
+                print >>sys.stderr, "Added new project: %s" % p.toxml()
+
+        first = False
+
+    return manifest
+
+
+def dump(manifest, out):
+
+    def sort_by_name(n1, n2):
+        return cmp(n1.getAttribute("name"), n2.getAttribute("name"))
+
+    out.write("""\
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+
+""")
+    for r in sorted(manifest.get_remotes(), sort_by_name):
+        out.write(r.toxml())
+        out.write("\n")
+    out.write("\n")
+
+    out.write(manifest.get_default().toxml())
+    out.write("\n\n")
+
+    for p in sorted(manifest.get_projects(), sort_by_name):
+        out.write(p.toxml())
+        out.write("\n")
+    out.write("""\
+
+</manifest>
+""")
+
+optparser = optparse.OptionParser(usage="%prog -o <union.xml> <manifest.xml>...")
+optparser.add_option("-o", metavar="FILE", help="Output file")
+options, args = optparser.parse_args(sys.argv[1:])
+if len(args) < 1:
+    optparser.error("Wrong number of arguments")
+
+m = union(args)
+out = sys.stdout
+if options.o:
+    out = open(options.o, "w")
+dump(m, out)
diff --git a/utils/seeded-builds/seed-update b/utils/seeded-builds/seed-update
new file mode 100755
index 0000000..65b391f
--- /dev/null
+++ b/utils/seeded-builds/seed-update
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Update and repack an Android Build seed tarball.
+# This is intended to be run from a crontab, once a week or so
+# (takes ~1hr to run, produces ~15Gb tarball as of writing).
+#
+
+set -e
+
+BASE_DIR=/mnt
+OUT_DIR=/mnt2/seed
+SEED=$1
+
+cd $BASE_DIR/$SEED
+time repo sync --quiet -j2
+cd $BASE_DIR
+time tar c $SEED/ | gzip -c -1 >$OUT_DIR/$SEED.tar.gz.new
+[ -f $OUT_DIR/$SEED.tar.gz ] && mv $OUT_DIR/$SEED.tar.gz $OUT_DIR/$SEED.tar.gz.old
+mv $OUT_DIR/$SEED.tar.gz.new $OUT_DIR/$SEED.tar.gz