aos/build/build.py

#!/usr/bin/python3

import argparse
import sys
import subprocess
import re
import os
import os.path
import string
import shutil
import errno
import queue
import threading

class TestThread(threading.Thread):
  """Runs 1 test and keeps track of its current state.

  A TestThread is either waiting to start the test, actually running it, done,
  running it, or stopped. The first 3 always happen in that order and can
  change to stopped at any time.

  It will finish (ie join() will return) once the process has exited, at which
  point accessing process to see the status is OK.

  Attributes:
    executable: The file path of the executable to run.
    env: The environment variables to set.
    done_queue: A queue.Queue to place self on once done running the test.
    start_semaphore: A threading.Semaphore to wait on before starting.
    process_lock: A lock around process.
    process: The currently executing test process or None. Synchronized by
        process_lock.
    stopped: True if we're stopped.
  """
  def __init__(self, executable, env, done_queue, start_semaphore):
    super(TestThread, self).__init__(
        name=os.path.split(executable)[1])

    self.executable = executable
    self.env = env
    self.done_queue = done_queue
    self.start_semaphore = start_semaphore

    self.process_lock = threading.Lock()
    self.process = None
    self.stopped = False

  def run(self):
    with self.start_semaphore:
      if self.stopped: return
      test_output('Starting test %s...' % self.name)
      self.output, subprocess_output = os.pipe()
      with self.process_lock:
        self.process = subprocess.Popen((self.executable,
                                         '--gtest_color=yes'),
                                        env=self.env,
                                        stderr=subprocess.STDOUT,
                                        stdout=subprocess_output,
                                        stdin=open(os.devnull, 'r'))
      os.close(subprocess_output)
      self.process.wait()
      with self.process_lock:
        self.returncode = self.process.returncode
        self.process = None
        if not self.stopped:
          self.done_queue.put(self)

  def terminate_process(self):
    """Asks any currently running process to stop.

    Also changes this object to the stopped state.
    """
    with self.process_lock:
      self.stopped = True
      if not self.process: return
      self.process.terminate()
  def kill_process(self):
    """Forcibly terminates any running process.

    Also changes this object to the stopped state.
    """
    with self.process_lock:
      self.stopped = True
      if not self.process: return
      self.process.kill()

def aos_path():
  """Returns:
    A relative path to the aos directory.
  """
  return os.path.join(os.path.dirname(__file__), '..')

def get_ip(device):
  """Retrieves the IP address for a given device."""
  FILENAME = os.path.normpath(os.path.join(aos_path(), '..',
                              'output', 'ip_base.txt'))
  if not os.access(FILENAME, os.R_OK):
    os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
    with open(FILENAME, 'w') as f:
      f.write('10.9.71')
  with open(FILENAME, 'r') as f:
    base = f.readline()
  if device == 'prime':
    return base + '.179'
  elif device == 'robot':
    return base + '.2'
  else:
    raise Exception('Unknown device %s to get an IP address for.' % device)

def user_output(message):
  """Prints message to the user."""
  print('build.py: ' + message, file=sys.stderr)

"""A lock to avoid making a mess intermingling test-related messages."""
test_output_lock = threading.RLock()
def test_output(message):
  """Prints message to the user. Intended for messages related to tests."""
  with test_output_lock:
    print('tests: ' + message, file=sys.stdout)

def call_download_externals(argument):
  """Calls download_externals.sh for a given set of externals.

  Args:
    argument: The argument to pass to the shell script to tell it what to
        download.
  """
  subprocess.check_call(
      (os.path.join(aos_path(), 'build', 'download_externals.sh'),
       argument),
      stdin=open(os.devnull, 'r'))

class Processor(object):
  """Represents a processor architecture we can build for."""

  class UnknownPlatform(Exception):
    def __init__(self, message):
      self.message = message

  class Platform(object):
    """Represents a single way to build the code."""

    def outdir(self):
      """Returns:
        The path of the directory build outputs get put in to.
      """
      return os.path.join(
          aos_path(), '..', 'output', self.outname())
    def build_ninja(self):
      """Returns:
        The path of the build.ninja file.
      """
      return os.path.join(self.outdir(), 'build.ninja')

    def do_deploy(self, dry_run, command):
      """Helper for subclasses to implement deploy.

      Args:
        dry_run: If True, prints the command instead of actually running it.
        command: A tuple of command-line arguments.
      """
      real_command = (('echo',) + command) if dry_run else command
      subprocess.check_call(real_command, stdin=open(os.devnull, 'r'))

    def deploy(self, dry_run):
      """Downloads the compiled code to the target computer."""
      raise NotImplementedError('deploy should be overriden')
    def outname(self):
      """Returns:
        The name of the directory the code will be compiled to.
      """
      raise NotImplementedError('outname should be overriden')
    def os(self):
      """Returns:
        The name of the operating system this platform is for.

        This will be used as the value of the OS gyp variable.
      """
      raise NotImplementedError('os should be overriden')
    def gyp_platform(self):
      """Returns:
        The platform name the .gyp files know.

        This will be used as the value of the PLATFORM gyp variable.
      """
      raise NotImplementedError('gyp_platform should be overriden')
    def architecture(self):
      """Returns:
        The processor architecture for this platform.

        This will be used as the value of the ARCHITECTURE gyp variable.
      """
      raise NotImplementedError('architecture should be overriden')
    def compiler(self):
      """Returns:
        The compiler used for this platform.

        Everything before the first _ will be used as the value of the
        COMPILER gyp variable and the whole thing will be used as the value
        of the FULL_COMPILER gyp variable.
      """
      raise NotImplementedError('compiler should be overriden')
    def debug(self):
      """Returns:
        Whether or not this platform compiles with debugging information.

        The DEBUG gyp variable will be set to "yes" or "no" based on this.
      """
      raise NotImplementedError('debug should be overriden')
    def build_env(self):
      """Returns:
        A map of environment variables to set while building this platform.
      """
      raise NotImplementedError('build_env should be overriden')

  def check_installed(self, platforms, is_deploy):
    """Makes sure that everything necessary to build platforms are installed."""
    raise NotImplementedError('check_installed should be overriden')
  def parse_platforms(self, string):
    """Args:
      string: A user-supplied string saying which platforms to select.

    Returns:
      A tuple of Platform objects.

    Raises:
      Processor.UnknownPlatform: Parsing string didn't work out.
    """
    raise NotImplementedError('parse_platforms should be overriden')
  def extra_gyp_flags(self):
    """Returns:
      A tuple of extra flags to pass to gyp (if any).
    """
    return ()
  def modify_ninja_file(self, ninja_file):
    """Modifies a freshly generated ninja file as necessary.

    Args:
      ninja_file: Path to the file to modify.
    """
    pass
  def download_externals(self, platforms):
    """Calls download_externals as appropriate to build platforms.

    Args:
      platforms: A list of platforms to download external libraries for.
    """
    raise NotImplementedError('download_externals should be overriden')

  # TODO(brians): Verify that this (and its callers) catch everything from a
  # fresh install.
  def do_check_installed(self, other_packages):
    """Helper for subclasses to implement check_installed.

    Args:
      other_packages: A tuple of platform-specific packages to check for."""
    all_packages = other_packages
    # Necessary to build stuff.
    all_packages += ('ccache', 'make')
    # Necessary to download stuff to build.
    all_packages += ('wget', 'git', 'subversion', 'patch', 'unzip', 'bzip2')
    # Necessary to build externals stuff.
    all_packages += ('python', 'gcc', 'g++')
    try:
      result = subprocess.check_output(
          ('dpkg-query', '--show') + all_packages,
          stdin=open(os.devnull, 'r'),
          stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
      output = e.output.decode('utf-8').rstrip()
      not_found = []
      for line in output.splitlines(True):
        match = re.match(r'dpkg-query: no packages found matching (.*)',
                         line)
        if match:
          not_found.append(match.group(1))
      user_output('Some packages not installed: %s.' % ', '.join(not_found))
      user_output('Try something like `sudo apt-get install %s`.' %
          ' '.join(not_found))
      exit(1)

class CRIOProcessor(Processor):
  """A Processor subclass for building cRIO code."""

  class Platform(Processor.Platform):
    def __init__(self, debug, wind_base):
      super(CRIOProcessor.Platform, self).__init__()

      self.__debug = debug
      self.__wind_base = wind_base

    def __repr__(self):
      return 'CRIOProcessor.Platform(debug=%s)' % self.debug()
    def __str__(self):
      return 'crio%s' % ('-debug' if self.debug() else '')

    def outname(self):
      return 'crio-debug' if self.debug() else 'crio'
    def os(self):
      return 'vxworks'
    def gyp_platform(self):
      return 'crio'
    def architecture(self):
      return 'ppc'
    def compiler(self):
      return 'gcc'
    def sanitizer(self):
      return 'none'
    def debug(self):
      return self.__debug
    def wind_base(self):
      return self.__wind_base

    # TODO(brians): test this
    def deploy(self, dry_run):
      self.do_deploy(dry_run,
                     ('ncftpput', get_ip('robot'), '/',
                      os.path.join(self.outdir(), 'lib', 'FRC_UserProgram.out')))

    def build_env(self):
      return {'WIND_BASE': self.wind_base()}

  def __init__(self):
    super(CRIOProcessor, self).__init__()

    if 'WIND_BASE' in os.environ:
      self.__wind_base = os.environ['WIND_BASE']
    else:
      self.__wind_base = '/usr/local/powerpc-wrs-vxworks/wind_base'

  def parse_platforms(self, string):
    if string is None or string == 'crio':
      return (CRIOProcessor.Platform(False, self.wind_base()),)
    elif string == 'crio-debug' or string == 'debug':
      return (CRIOProcessor.Platform(True, self.wind_base()),)
    else:
      raise Processor.UnknownPlatform('Unknown cRIO platform "%s".' % string)

  def wind_base(self):
    return self.__wind_base

  def extra_gyp_flags(self):
    return ('-DWIND_BASE=%s' % self.wind_base(),)

  def modify_ninja_file(self, ninja_file):
    subprocess.check_call(
        ('sed', '-i',
         's/nm -gD/nm/g', ninja_file),
        stdin=open(os.devnull, 'r'))

  def download_externals(self, _):
    call_download_externals('crio')

  def check_installed(self, platforms, is_deploy):
    packages = ('powerpc-wrs-vxworks', 'tcl')
    if is_deploy:
      packages += ('ncftp',)
    self.do_check_installed(packages)

class PrimeProcessor(Processor):
  """A Processor subclass for building prime code."""

  class Platform(Processor.Platform):
    def __init__(self, architecture, compiler, debug, sanitizer):
      super(PrimeProcessor.Platform, self).__init__()

      self.__architecture = architecture
      self.__compiler = compiler
      self.__debug = debug
      self.__sanitizer = sanitizer

    def __repr__(self):
      return 'PrimeProcessor.Platform(architecture=%s, compiler=%s, debug=%s' \
          ', sanitizer=%s)' \
          % (self.architecture(), self.compiler(), self.debug(),
             self.sanitizer())
    def __str__(self):
      return '%s-%s%s-%s' % (self.architecture(), self.compiler(),
                          '-debug' if self.debug() else '', self.sanitizer())

    def os(self):
      return 'linux'
    def gyp_platform(self):
      return '%s-%s-%s' % (self.os(), self.architecture(), self.compiler())
    def architecture(self):
      return self.__architecture
    def compiler(self):
      return self.__compiler
    def sanitizer(self):
      return self.__sanitizer
    def debug(self):
      return self.__debug

    def outname(self):
      return str(self)

    # TODO(brians): test this
    def deploy(self, dry_run):
      # Downloads code to the prime in a way that avoids clashing too badly with
      # starter (like the naive download everything one at a time).
      SUM = 'md5sum'
      TARGET_DIR = '/home/driver/robot_code/bin'
      TEMP_DIR = '/tmp/aos_downloader'
      TARGET = 'driver@' + get_ip('prime')

      from_dir = os.path.join(self.outdir(), 'outputs')
      sums = subprocess.check_output((SUM,) + tuple(os.listdir(from_dir)),
                                     stdin=open(os.devnull, 'r'),
                                     cwd=from_dir)
      to_download = subprocess.check_output(
          ('ssh', TARGET,
           """rm -rf {TMPDIR} && mkdir {TMPDIR} && cd {TO_DIR}
             && echo '{SUMS}' | {SUM} --check --quiet
             |& grep -F FAILED | sed 's/^\\(.*\\): FAILED.*"'$'"/\\1/g'""".format(
               TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR, SUMS=sums, SUM=SUM)))
      if not to_download:
        user_output("Nothing to download")
        return
      self.do_deploy(
          dry_run,
          ('scp', '-o', 'Compression yes') + to_download
          + (('%s:%s' % (TARGET, TEMP_DIR)),))
      if not dry_run:
        subprocess.check_call(
            ('ssh', TARGET,
             """mv {TMPDIR}/* {TO_DIR}
             && echo 'Done moving new executables into place'
             && ionice -c 3 bash -c 'sync && sync && sync'""".format(
                 TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR)))

    def build_env(self):
      OTHER_SYSROOT = '/opt/clang-3.5/'
      SYMBOLIZER_PATH = OTHER_SYSROOT + 'bin/llvm-symbolizer'
      r = {}
      if self.compiler() == 'clang' or self.compiler() == 'gcc_4.8':
        r['LD_LIBRARY_PATH'] = OTHER_SYSROOT + 'lib64'
      if self.sanitizer() == 'address':
        r['ASAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH
        r['ASAN_OPTIONS'] = 'detect_leaks=1:check_initialization_order=1:strict_init_order=1'
      elif self.sanitizer() == 'memory':
        r['MSAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH
      elif self.sanitizer() == 'thread':
        r['TSAN_OPTIONS'] = 'external_symbolizer_path=' + SYMBOLIZER_PATH

      r['CCACHE_COMPRESS'] = 'yes'
      r['CCACHE_DIR'] = \
          os.path.abspath(os.path.join(aos_path(), '..', 'output', 'ccache_dir'))
      r['CCACHE_HASHDIR'] = 'yes'
      if self.compiler() == 'clang':
        # clang doesn't like being run directly on the preprocessed files.
        r['CCACHE_CPP2'] = 'yes'
      # Without this, ccache slows down because of the generated header files.
      # The race condition that this opens up isn't a problem because the build
      # system finishes modifying header files before compiling anything that
      # uses them.
      r['CCACHE_SLOPPINESS'] = 'include_file_mtime'

      if self.architecture() == 'amd64':
        r['PATH'] = os.path.join(aos_path(), 'build', 'bin-ld.gold') + \
            ':' + os.environ['PATH']

      return r

  ARCHITECTURES = ('arm', 'amd64')
  COMPILERS = ('clang', 'gcc', 'gcc_4.8')
  SANITIZERS = ('address', 'undefined', 'integer', 'memory', 'thread', 'none')
  SANITIZER_TEST_WARNINGS = {
      'memory': (True,
"""We don't have all of the libraries instrumented which leads to lots of false
  errors with msan (especially stdlibc++).
  TODO(brians): Figure out a way to deal with it."""),
      'undefined': (False,
"""There are several warnings in other people's code that ubsan catches.
  The following have been verified non-interesting:
    include/c++/4.8.2/array:*: runtime error: reference binding to null pointer of type 'int'
      This happens with ::std::array<T, 0> and it doesn't seem to cause any issues.
    output/downloaded/eigen-3.2.1/Eigen/src/Core/util/Memory.h:782:*: runtime error: load of misaligned address 0x* for type 'const int', which requires 4 byte alignment
      That's in the CPUID detection code which only runs on x86."""),
  }
  PIE_SANITIZERS = ('memory', 'thread')

  def __init__(self, is_test, is_deploy):
    super(Processor, self).__init__()

    platforms = []
    for architecture in PrimeProcessor.ARCHITECTURES:
      for compiler in PrimeProcessor.COMPILERS:
        for debug in [True, False]:
          if architecture == 'arm' and compiler == 'gcc_4.8':
            # We don't have a compiler to use here.
            continue
          platforms.append(
              PrimeProcessor.Platform(architecture, compiler, debug, 'none'))
    for sanitizer in PrimeProcessor.SANITIZERS:
      for compiler in ('gcc_4.8', 'clang'):
        if compiler == 'gcc_4.8' and (sanitizer == 'undefined' or
                                      sanitizer == 'integer' or
                                      sanitizer == 'memory'):
          # GCC 4.8 doesn't support these sanitizers.
          continue
        if sanitizer == 'none':
          # We already added sanitizer == 'none' above.
          continue
        platforms.append(
            PrimeProcessor.Platform('amd64', compiler, True, sanitizer))
    self.__platforms = frozenset(platforms)

    if is_test:
      default_platforms = self.select_platforms(architecture='amd64',
                                                debug=True)
      for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items():
        if warning[0]:
          default_platforms -= self.select_platforms(sanitizer=sanitizer)
    elif is_deploy:
      # TODO(brians): Switch to deploying the code built with clang.
      default_platforms = self.select_platforms(architecture='arm',
                                                compiler='gcc',
                                                debug=False)
    else:
      default_platforms = self.select_platforms(debug=False)
    self.__default_platforms = frozenset(default_platforms)

  def platforms(self):
    return self.__platforms
  def default_platforms(self):
    return self.__default_platforms

  def download_externals(self, platforms):
    to_download = set()
    for architecture in PrimeProcessor.ARCHITECTURES:
      for sanitizer in PrimeProcessor.PIE_SANITIZERS:
        if platforms & self.select_platforms(architecture=architecture,
                                             sanitizer=sanitizer):
          to_download.add(architecture + '-fPIE')
        if platforms & self.select_platforms(architecture=architecture,
                                             sanitizer='none'):
          to_download.add(architecture)
    for download_target in to_download:
      call_download_externals(download_target)

  def parse_platforms(self, string):
    if string is None:
      return self.default_platforms()
    elif string == 'all':
      return self.platforms()
    r = self.default_platforms()
    for part in string.split(','):
      if part[0] == '+':
        r = r | self.select_platforms_string(part[1:])
      elif part[0] == '-':
        r = r - self.select_platforms_string(part[1:])
      elif part[0] == '=':
        r = self.select_platforms_string(part[1:])
      else:
        selected = self.select_platforms_string(part)
        r = r - (self.platforms() - selected)
        if not r:
          r = selected
    return r

  def select_platforms(self, architecture=None, compiler=None, debug=None, sanitizer=None):
    r = []
    for platform in self.platforms():
      if architecture is None or platform.architecture() == architecture:
        if compiler is None or platform.compiler() == compiler:
          if debug is None or platform.debug() == debug:
            if sanitizer is None or platform.sanitizer() == sanitizer:
              r.append(platform)
    return set(r)

  def select_platforms_string(self, string):
    r = []
    architecture, compiler, debug, sanitizer = None, None, None, None
    for part in string.split('-'):
      if part in PrimeProcessor.ARCHITECTURES:
        architecture = part
      elif part in PrimeProcessor.COMPILERS:
        compiler = part
      elif part in ['debug', 'dbg']:
        debug = True
      elif part in ['release', 'nodebug', 'ndb']:
        debug = False
      elif part in PrimeProcessor.SANITIZERS:
        sanitizer = part
      else:
        raise Processor.UnknownPlatform('Unknown platform string component "%s".' % part)
    return self.select_platforms(
        architecture=architecture,
        compiler=compiler,
        debug=debug,
        sanitizer=sanitizer)

  def check_installed(self, platforms, is_deploy):
    packages = set(('lzip', 'm4', 'realpath'))
    packages.add('ruby')
    # clang-format from here gets used for all versions.
    packages.add('clang-3.5')
    packages.add('arm-eabi-gcc')
    for platform in platforms:
      if platform.architecture() == 'arm':
        packages.add('gcc-4.7-arm-linux-gnueabihf')
        packages.add('g++-4.7-arm-linux-gnueabihf')
      if platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8':
        packages.add('clang-3.5')
      if is_deploy:
        packages.add('openssh-client')
      if platform.compiler() == 'gcc' and platform.architecture() == 'amd64':
        packages.add('gcc-4.7')
        packages.add('g++-4.7')

    self.do_check_installed(tuple(packages))

def main():
  class TryParsingAgain(Exception):
    pass

  class TryAgainArgumentParser(argparse.ArgumentParser):
    def __init__(self, **kwargs):
      super(TryAgainArgumentParser, self).__init__(**kwargs)

    def error(self, message):
      raise TryParsingAgain

  def set_up_parser(parser, args):
    def add_build_args(parser):
      parser.add_argument(
          'target',
          help='target to build',
          nargs='*')
      parser.add_argument(
          '--jobs', '-j',
          help='number of things to do at once',
          type=int)
    def add_common_args(parser):
      parser.add_argument(
          'platforms',
          help='platform(s) to act on',
          nargs='?')

    parser.add_argument('--processor', required=True, help='prime or crio')
    parser.add_argument('--main_gyp', required=True, help='main .gyp file')
    subparsers = parser.add_subparsers(dest='action_name')

    build_parser = subparsers.add_parser(
        'build',
        help='build the code (default)')
    add_common_args(build_parser)
    add_build_args(build_parser)

    clean_parser = subparsers.add_parser(
        'clean',
        help='remove all output directories')
    add_common_args(clean_parser)

    deploy_parser = subparsers.add_parser(
        'deploy',
        help='build and download the code')
    add_common_args(deploy_parser)
    add_build_args(deploy_parser)
    deploy_parser.add_argument(
      '-n', '--dry-run',
      help="don't actually download anything",
      action='store_true')

    tests_parser = subparsers.add_parser(
        'tests',
        help='run tests')
    add_common_args(tests_parser)
    add_build_args(tests_parser)

    return parser.parse_args(args)

  try:
    parser = TryAgainArgumentParser()
    args = set_up_parser(parser, sys.argv[1:])
  except TryParsingAgain:
    parser = argparse.ArgumentParser()
    REQUIRED_ARGS_END = 5
    args = set_up_parser(parser, sys.argv[1:REQUIRED_ARGS_END] + ['build'] +
                         sys.argv[(REQUIRED_ARGS_END):])

  if args.processor == 'crio':
    processor = CRIOProcessor()
  elif args.processor == 'prime':
    processor = PrimeProcessor(args.action_name == 'tests',
                               args.action_name == 'deploy')
  else:
    parser.exit(status=1, message='Unknown processor "%s".' % args.processor)

  if 'target' in args:
    targets = args.target[:]
  else:
    targets = []
  unknown_platform_error = None
  try:
    platforms = processor.parse_platforms(args.platforms)
  except Processor.UnknownPlatform as e:
    unknown_platform_error = e.message
    targets.append(args.platforms)
    platforms = processor.parse_platforms(None)
  if not platforms:
    user_output("No platforms selected!")
    exit(1)

  processor.check_installed(platforms, args.action_name == 'deploy')
  processor.download_externals(platforms)

  class ToolsConfig(object):
    def __init__(self):
      self.variables = {'AOS': aos_path()}
      with open(os.path.join(aos_path(), 'build', 'tools_config'), 'r') as f:
        for line in f:
          if line[0] == '#':
            pass
          elif line.isspace():
            pass
          else:
            new_name, new_value = line.rstrip().split('=')
            for name, value in self.variables.items():
              new_value = new_value.replace('${%s}' % name, value)
            self.variables[new_name] = new_value
    def __getitem__(self, key):
      return self.variables[key]

  tools_config = ToolsConfig()

  def handle_clean_error(function, path, excinfo):
    if issubclass(OSError, excinfo[0]):
      if excinfo[1].errno == errno.ENOENT:
        # Who cares if the file we're deleting isn't there?
        return
    raise excinfo[1]

  def need_to_run_gyp(platform):
    """Determines if we need to run gyp again or not.

    The generated build files are supposed to re-run gyp again themselves, but
    that doesn't work (or at least it used to not) and we sometimes want to
    modify the results anyways.

    Args:
      platform: The platform to check for.
    """
    if not os.path.exists(platform.build_ninja()):
      return True
    if os.path.getmtime(__file__) > os.path.getmtime(platform.build_ninja()):
      return True
    dirs = os.listdir(os.path.join(aos_path(), '..'))
    # Looking through these folders takes a long time and isn't useful.
    if dirs.count('output'): dirs.remove('output')
    if dirs.count('.git'): dirs.remove('.git')
    return not not subprocess.check_output(
        ('find',) + tuple(os.path.join(aos_path(), '..', d) for d in dirs)
         + ('-newer', platform.build_ninja(),
         '(', '-name', '*.gyp', '-or', '-name', '*.gypi', ')'),
        stdin=open(os.devnull, 'r'))

  def env(platform):
    """Makes sure we pass through important environmental variables.

    Returns:
      An environment suitable for passing to subprocess.Popen and friends.
    """
    build_env = dict(platform.build_env())
    if not 'TERM' in build_env:
      build_env['TERM'] = os.environ['TERM']
    if not 'PATH' in build_env:
      build_env['PATH'] = os.environ['PATH']
    return build_env

  to_build = []
  for platform in platforms:
    to_build.append(str(platform))
  if len(to_build) > 1:
    to_build[-1] = 'and ' + to_build[-1]
  user_output('Building %s...' % ', '.join(to_build))

  if args.action_name == 'tests':
    for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items():
      warned_about = platforms & processor.select_platforms(sanitizer=sanitizer)
      if warned_about:
        user_output(warning[1])
        if warning[0]:
          # TODO(brians): Add a --force flag or something to override this?
          user_output('Refusing to run tests for sanitizer %s.' % sanitizer)
          exit(1)

  num = 1
  for platform in platforms:
    user_output('Building %s (%d/%d)...' % (platform, num, len(platforms)))
    if args.action_name == 'clean':
      shutil.rmtree(platform.outdir(), onerror=handle_clean_error)
    else:
      if need_to_run_gyp(platform):
        user_output('Running gyp...')
        gyp = subprocess.Popen(
            (tools_config['GYP'],
             '--check',
             '--depth=%s' % os.path.join(aos_path(), '..'),
             '--no-circular-check',
             '-f', 'ninja',
             '-I%s' % os.path.join(aos_path(), 'build', 'aos.gypi'),
             '-I/dev/stdin', '-Goutput_dir=output',
             '-DOS=%s' % platform.os(),
             '-DPLATFORM=%s' % platform.gyp_platform(),
             '-DARCHITECTURE=%s' % platform.architecture(),
             '-DCOMPILER=%s' % platform.compiler().split('_')[0],
             '-DFULL_COMPILER=%s' % platform.compiler(),
             '-DDEBUG=%s' % ('yes' if platform.debug() else 'no'),
             '-DSANITIZER=%s' % platform.sanitizer(),
             '-DSANITIZER_FPIE=%s' % ('-fPIE'
                 if platform.sanitizer() in PrimeProcessor.PIE_SANITIZERS
                 else '')) +
            processor.extra_gyp_flags() + (args.main_gyp,),
            stdin=subprocess.PIPE)
        gyp.communicate(("""
{
  'target_defaults': {
    'configurations': {
      '%s': {}
    }
  }
}""" % platform.outname()).encode())
        if gyp.returncode:
          user_output("Running gyp failed!")
          exit(1)
        processor.modify_ninja_file(platform.build_ninja())
        user_output('Done running gyp')
      else:
        user_output("Not running gyp")

      try:
        call = (tools_config['NINJA'],
                '-C', platform.outdir()) + tuple(targets)
        if args.jobs:
          call += ('-j', str(args.jobs))
        subprocess.check_call(call,
            stdin=open(os.devnull, 'r'),
            env=env(platform))
      except subprocess.CalledProcessError as e:
        if unknown_platform_error is not None:
          user_output(unknown_platform_error)
        raise e

    if args.action_name == 'deploy':
      platform.deploy(args.dry_run)
    elif args.action_name == 'tests':
      dirname = os.path.join(platform.outdir(), 'tests')
      done_queue = queue.Queue()
      running = []
      if args.jobs:
        number_jobs = args.jobs
      else:
        number_jobs = os.sysconf('SC_NPROCESSORS_ONLN') + 2
      test_start_semaphore = threading.Semaphore(number_jobs)
      if targets:
        to_run = []
        for target in targets:
          if target.endswith('_test'):
            to_run.append(target)
      else:
        to_run = os.listdir(dirname)
      for f in to_run:
        thread = TestThread(os.path.join(dirname, f), env(platform), done_queue,
                            test_start_semaphore)
        running.append(thread)
        thread.start()
      try:
        while running:
          done = done_queue.get()
          running.remove(done)
          with test_output_lock:
            test_output('Output from test %s:' % done.name)
            for line in os.fdopen(done.output):
              if not sys.stdout.isatty():
                # Remove color escape codes.
                line = re.sub(r'\x1B\[[0-9;]*[a-zA-Z]', '', line)
              sys.stdout.write(line)
            if not done.returncode:
              test_output('Test %s succeeded' % done.name)
            else:
              test_output('Test %s failed' % done.name)
              user_output('Aborting because of test failure.')
              exit(1)
      finally:
        if running:
          test_output('Killing other tests...')
          for thread in running:
            thread.terminate_process()
          to_remove = []
          for thread in running:
            thread.join(5)
            if not thread.is_alive():
              to_remove.append(thread);
          for thread in to_remove: running.remove(thread)
          for thread in running:
            test_output(
                'Test %s did not terminate. Killing it.' % thread.name)
            thread.kill_process()
            thread.join()
          test_output('Done killing other tests')

    user_output('Done building %s (%d/%d)' % (platform, num, len(platforms)))
    num += 1

if __name__ == '__main__':
  main()