scripts: Refactor build-all.py's build

Build all consists of a mismash of figuring out variables, running
shutils, and actually executing commands.  In preparation for parallel
builds, change this to build up the commands in advance, and then have
a tracker go through and actually run the commands.

Change-Id: I24215b14e5d045f8269bbdc2af9989ab21b2310b
Signed-off-by: David Brown <davidb@codeaurora.org>

1 files changed, 133 insertions, 62 deletions

diff --git a/scripts/build-all.py b/scripts/build-all.py
index 189be5c7b76..740f11386ff 100755
--- a/scripts/build-all.py
+++ b/scripts/build-all.py
@@ -28,6 +28,7 @@
 
 # Build the kernel for all targets using the Android build environment.
 
+from collections import namedtuple
 import glob
 from optparse import OptionParser
 import os
@@ -73,44 +74,122 @@ def check_build():
 
 failed_targets = []
 
-class LogRunner:
-    def __init__(self, logname, make_env):
-        self.logname = logname
-        self.fd = open(logname, 'w')
-        self.make_env = make_env
-
-    def run(self, args):
-        devnull = open('/dev/null', 'r')
-        proc = subprocess.Popen(args, stdin=devnull,
-                env=self.make_env,
-                bufsize=0,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT)
-        count = 0
-        # for line in proc.stdout:
-        rawfd = proc.stdout.fileno()
-        while True:
-            line = os.read(rawfd, 1024)
-            if not line:
+class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])):
+
+    def set_width(self, width):
+        self.width = width
+
+    def __enter__(self):
+        print "Building:", self.short_name
+        self.log = open(self.log_name, 'w')
+    def __exit__(self, type, value, traceback):
+        self.log.close()
+
+    def run(self):
+        self.status = None
+        def printer(line):
+            text = "[%-*s] %s" % (self.width, self.short_name, line)
+            print text
+            self.log.write(text)
+            self.log.write('\n')
+        for step in self.steps:
+            st = step.run(printer)
+            if st:
+                self.status = st
                 break
-            self.fd.write(line)
-            self.fd.flush()
-            if all_options.verbose:
-                sys.stdout.write(line)
-                sys.stdout.flush()
+
+class BuildTracker:
+    """Manages all of the steps necessary to perform a build.  The
+    build consists of one or more sequences of steps.  The different
+    sequences can be processed independently, while the steps within a
+    sequence must be done in order."""
+
+    def __init__(self):
+        self.sequence = []
+
+    def add_sequence(self, log_name, short_name, steps):
+        self.sequence.append(BuildSequence(log_name, short_name, steps))
+
+    def longest_name(self):
+        longest = 0
+        for seq in self.sequence:
+            longest = max(longest, len(seq.short_name))
+        return longest
+
+    def __repr__(self):
+        return "BuildTracker(%s)" % self.sequence
+
+    def run(self):
+        longest = self.longest_name()
+        errors = []
+        for seq in self.sequence:
+            seq.set_width(longest)
+            with seq:
+                seq.run()
+                if seq.status:
+                    errors.append(seq.short_name)
+        if errors:
+            fail("\n  ".join(["Failed targets:"] + errors))
+
+class PrintStep:
+    """A step that just prints a message"""
+    def __init__(self, message):
+        self.message = message
+
+    def run(self, outp):
+        outp(self.message)
+
+class MkdirStep:
+    """A step that makes a directory"""
+    def __init__(self, direc):
+        self.direc = direc
+
+    def run(self, outp):
+        outp("mkdir %s" % self.direc)
+        os.mkdir(self.direc)
+
+class RmtreeStep:
+    def __init__(self, direc):
+        self.direc = direc
+
+    def run(self, outp):
+        outp("rmtree %s" % self.direc)
+        shutil.rmtree(self.direc, ignore_errors=True)
+
+class CopyfileStep:
+    def __init__(self, src, dest):
+        self.src = src
+        self.dest = dest
+
+    def run(self, outp):
+        outp("cp %s %s" % (self.src, self.dest))
+        shutil.copyfile(self.src, self.dest)
+
+class ExecStep:
+    def __init__(self, cmd, **kwargs):
+        self.cmd = cmd
+        self.kwargs = kwargs
+
+    def run(self, outp):
+        outp("exec: %s" % (" ".join(self.cmd),))
+        with open('/dev/null', 'r') as devnull:
+            proc = subprocess.Popen(self.cmd, stdin=devnull,
+                    bufsize=0,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    **self.kwargs)
+            stdout = proc.stdout
+            while True:
+                line = stdout.readline()
+                if not line:
+                    break
+                line = line.rstrip('\n')
+                outp(line)
+            result = proc.wait()
+            if result != 0:
+                return ('error', result)
             else:
-                for i in range(line.count('\n')):
-                    count += 1
-                    if count == 64:
-                        count = 0
-                        print
-                    sys.stdout.write('.')
-                sys.stdout.flush()
-        print
-        result = proc.wait()
-
-        self.fd.flush()
-        return result
+                return None
 
 class Builder():
 
@@ -132,27 +211,27 @@ class Builder():
         else:
             self.make_env['ARCH'] = 'arm'
         self.make_env['KCONFIG_NOTIMESTAMP'] = 'true'
+        self.log_name = "%s/log-%s.log" % (build_dir, self.name)
 
     def build(self):
+        steps = []
         dest_dir = os.path.join(build_dir, self.name)
         log_name = "%s/log-%s.log" % (build_dir, self.name)
-        print 'Building %s in %s log %s' % (self.name, dest_dir, log_name)
+        steps.append(PrintStep('Building %s in %s log %s' %
+            (self.name, dest_dir, log_name)))
         if not os.path.isdir(dest_dir):
-            os.mkdir(dest_dir)
+            steps.append(MkdirStep(dest_dir))
         defconfig = self.defconfig
         dotconfig = '%s/.config' % dest_dir
         savedefconfig = '%s/defconfig' % dest_dir
-        # shutil.copyfile(defconfig, dotconfig)  # Not really right.
 
         staging_dir = 'install_staging'
         modi_dir = '%s' % staging_dir
         hdri_dir = '%s/usr' % staging_dir
-        shutil.rmtree(os.path.join(dest_dir, staging_dir), ignore_errors=True)
+        steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir)))
 
-        with open('/dev/null', 'r') as devnull:
-            subprocess.check_call(['make', 'O=%s' % dest_dir,
-                self.confname], env=self.make_env,
-                stdin=devnull)
+        steps.append(ExecStep(['make', 'O=%s' % dest_dir,
+            self.confname], env=self.make_env))
 
         if not all_options.updateconfigs:
             # Build targets can be dependent upon the completion of
@@ -167,24 +246,16 @@ class Builder():
                     cmd_line.append(c)
                 else:
                     build_targets.append(c)
-            build = LogRunner(log_name, self.make_env)
             for t in build_targets:
-                result = build.run(cmd_line + [t])
-                if result != 0:
-                    if all_options.keep_going:
-                        failed_targets.append(target)
-                        fail_or_error = error
-                    else:
-                        fail_or_error = fail
-                    fail_or_error("Failed to build %s, see %s" %
-                            (t, build.logname))
+                steps.append(ExecStep(cmd_line + [t], env=self.make_env))
 
         # Copy the defconfig back.
         if all_options.configs or all_options.updateconfigs:
-            with open('/dev/null', 'r') as devnull:
-                subprocess.check_call(['make', 'O=%s' % dest_dir,
-                    'savedefconfig'], env=self.make_env, stdin=devnull)
-            shutil.copyfile(savedefconfig, defconfig)
+            steps.append(ExecStep(['make', 'O=%s' % dest_dir,
+                'savedefconfig'], env=self.make_env))
+            steps.append(CopyfileStep(savedefconfig, defconfig))
+
+        return steps
 
 def update_config(file, str):
     print 'Updating %s with \'%s\'\n' % (file, str)
@@ -217,13 +288,13 @@ def scan_configs():
 
 def build_many(targets):
     print "Building %d target(s)" % len(targets)
+    tracker = BuildTracker()
     for target in targets:
         if all_options.updateconfigs:
             update_config(target.defconfig, all_options.updateconfigs)
-        target.build()
-    if failed_targets:
-        fail("\n  ".join(["Failed targets:"] +
-            [target.name for target in failed_targets]))
+        steps = target.build()
+        tracker.add_sequence(target.log_name, target.name, steps)
+    tracker.run()
 
 def main():
     global make_command