aos/build/build.py

#!/usr/bin/python3

import argparse
import sys
import subprocess
import re
import os
import os.path
import string
import shutil
import errno
import queue
import threading
import pty
import signal

class TestThread(threading.Thread):
  """Runs 1 test and keeps track of its current state.

  A TestThread is either waiting to start the test, actually running it, done,
  running it, or stopped. The first 3 always happen in that order and can
  change to stopped at any time.

  It will finish (ie join() will return) once the process has exited, at which
  point accessing process to see the status is OK.

  Attributes:
    executable: The file path of the executable to run.
    env: The environment variables to set.
    done_queue: A queue.Queue to place self on once done running the test.
    start_semaphore: A threading.Semaphore to wait on before starting.
    process_lock: A lock around process.
    process: The currently executing test process or None. Synchronized by
        process_lock.
    stopped: True if we're stopped.
  """
  def __init__(self, executable, env, done_queue, start_semaphore):
    super(TestThread, self).__init__(
        name=os.path.split(executable)[1])

    self.executable = executable
    self.env = env
    self.done_queue = done_queue
    self.start_semaphore = start_semaphore

    self.process_lock = threading.Lock()
    self.process = None
    self.stopped = False
    self.returncode = None
    self.output = None

  def run(self):
    with self.start_semaphore:
      if self.stopped:
        return
      test_output('Starting test %s...' % self.name)
      self.output, subprocess_output = pty.openpty()
      try:
        with self.process_lock:
          self.process = subprocess.Popen(self.executable,
                                          env=self.env,
                                          stderr=subprocess.STDOUT,
                                          stdout=subprocess_output,
                                          stdin=open(os.devnull, 'r'))
      finally:
        os.close(subprocess_output)
      self.process.wait()
      with self.process_lock:
        self.returncode = self.process.returncode
        self.process = None
        if not self.stopped:
          self.done_queue.put(self)

  def terminate_process(self):
    """Asks any currently running process to stop."""
    with self.process_lock:
      if not self.process:
        return
      try:
        self.process.terminate()
      except OSError as e:
        if e.errno == errno.ESRCH:
          # We don't really care if it's already gone.
          pass
        else:
          raise e
  def kill_process(self):
    """Forcibly terminates any running process."""
    with self.process_lock:
      if not self.process:
        return
      try:
        self.process.kill()
      except OSError as e:
        if e.errno == errno.ESRCH:
          # We don't really care if it's already gone.
          pass
        else:
          raise e
  def stop(self):
    """Changes self to the stopped state."""
    with self.process_lock:
      self.stopped = True

def aos_path():
  """Returns:
    A relative path to the aos directory.
  """
  return os.path.join(os.path.dirname(__file__), '..')

def get_ip(device):
  """Retrieves the IP address for a given device."""
  FILENAME = os.path.normpath(os.path.join(aos_path(), '..',
                                           'output', 'ip_base.txt'))
  if not os.access(FILENAME, os.R_OK):
    os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
    with open(FILENAME, 'w') as f:
      f.write('10.9.71')
  with open(FILENAME, 'r') as f:
    base = f.readline()
  if device == 'prime':
    return base + '.179'
  elif device == 'robot':
    return base + '.2'
  else:
    raise Exception('Unknown device %s to get an IP address for.' % device)

def user_output(message):
  """Prints message to the user."""
  print('build.py: ' + message, file=sys.stderr)

# A lock to avoid making a mess intermingling test-related messages.
test_output_lock = threading.RLock()
def test_output(message):
  """Prints message to the user. Intended for messages related to tests."""
  with test_output_lock:
    print('tests: ' + message, file=sys.stdout)

def call_download_externals(argument):
  """Calls download_externals.sh for a given set of externals.

  Args:
    argument: The argument to pass to the shell script to tell it what to
        download.
  """
  subprocess.check_call(
      (os.path.join(aos_path(), 'build', 'download_externals.sh'),
       argument),
      stdin=open(os.devnull, 'r'))

class Processor(object):
  """Represents a processor architecture we can build for."""

  class UnknownPlatform(Exception):
    def __init__(self, message):
      super(Processor.UnknownPlatform, self).__init__()
      self.message = message

  class Platform(object):
    """Represents a single way to build the code."""

    def outdir(self):
      """Returns:
        The path of the directory build outputs get put in to.
      """
      return os.path.join(
          aos_path(), '..', 'output', self.outname())
    def build_ninja(self):
      """Returns:
        The path of the build.ninja file.
      """
      return os.path.join(self.outdir(), 'build.ninja')

    def do_deploy(self, dry_run, command):
      """Helper for subclasses to implement deploy.

      Args:
        dry_run: If True, prints the command instead of actually running it.
        command: A tuple of command-line arguments.
      """
      real_command = (('echo',) + command) if dry_run else command
      subprocess.check_call(real_command, stdin=open(os.devnull, 'r'))

    def deploy(self, dry_run):
      """Downloads the compiled code to the target computer."""
      raise NotImplementedError('deploy should be overriden')
    def outname(self):
      """Returns:
        The name of the directory the code will be compiled to.
      """
      raise NotImplementedError('outname should be overriden')
    def os(self):
      """Returns:
        The name of the operating system this platform is for.

        This will be used as the value of the OS gyp variable.
      """
      raise NotImplementedError('os should be overriden')
    def gyp_platform(self):
      """Returns:
        The platform name the .gyp files know.

        This will be used as the value of the PLATFORM gyp variable.
      """
      raise NotImplementedError('gyp_platform should be overriden')
    def architecture(self):
      """Returns:
        The processor architecture for this platform.

        This will be used as the value of the ARCHITECTURE gyp variable.
      """
      raise NotImplementedError('architecture should be overriden')
    def compiler(self):
      """Returns:
        The compiler used for this platform.

        Everything before the first _ will be used as the value of the
        COMPILER gyp variable and the whole thing will be used as the value
        of the FULL_COMPILER gyp variable.
      """
      raise NotImplementedError('compiler should be overriden')
    def sanitizer(self):
      """Returns:
        The sanitizer used on this platform.

        This will be used as the value of the SANITIZER gyp variable.

        "none" if there isn't one.
      """
      raise NotImplementedError('sanitizer should be overriden')
    def debug(self):
      """Returns:
        Whether or not this platform compiles with debugging information.

        The DEBUG gyp variable will be set to "yes" or "no" based on this.
      """
      raise NotImplementedError('debug should be overriden')
    def build_env(self):
      """Returns:
        A map of environment variables to set while building this platform.
      """
      raise NotImplementedError('build_env should be overriden')
    def priority(self):
      """Returns:
        A relative priority for this platform relative to other ones.

      Higher priority platforms will get built, tested, etc first. Generally,
      platforms which give higher-quality compiler errors etc should come first.
      """
      return 0

  def check_installed(self, platforms, is_deploy):
    """Makes sure that everything necessary to build platforms are installed."""
    raise NotImplementedError('check_installed should be overriden')
  def parse_platforms(self, platforms_string):
    """Args:
      string: A user-supplied string saying which platforms to select.

    Returns:
      A tuple of Platform objects.

    Raises:
      Processor.UnknownPlatform: Parsing string didn't work out.
    """
    raise NotImplementedError('parse_platforms should be overriden')
  def extra_gyp_flags(self):
    """Returns:
      A tuple of extra flags to pass to gyp (if any).
    """
    return ()
  def modify_ninja_file(self, ninja_file):
    """Modifies a freshly generated ninja file as necessary.

    Args:
      ninja_file: Path to the file to modify.
    """
    pass
  def download_externals(self, platforms):
    """Calls download_externals as appropriate to build platforms.

    Args:
      platforms: A list of platforms to download external libraries for.
    """
    raise NotImplementedError('download_externals should be overriden')

  def do_check_installed(self, other_packages):
    """Helper for subclasses to implement check_installed.

    Args:
      other_packages: A tuple of platform-specific packages to check for."""
    all_packages = other_packages
    # Necessary to build stuff.
    all_packages += ('ccache', 'make')
    # Necessary to download stuff to build.
    all_packages += ('wget', 'git', 'subversion', 'patch', 'unzip', 'bzip2')
    # Necessary to build externals stuff.
    all_packages += ('python', 'gcc', 'g++')
    try:
      # TODO(brians): Check versions too.
      result = subprocess.check_output(
          ('dpkg-query', '--show') + all_packages,
          stdin=open(os.devnull, 'r'),
          stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
      output = e.output.decode('utf-8').rstrip()
      not_found = []
      for line in output.splitlines(True):
        match = re.match(r'dpkg-query: no packages found matching (.*)',
                         line)
        if match:
          not_found.append(match.group(1))
      user_output('Some packages not installed: %s.' % ', '.join(not_found))
      user_output('Try something like `sudo apt-get install %s`.' %
                  ' '.join(not_found))
      exit(1)

class CRIOProcessor(Processor):
  """A Processor subclass for building cRIO code."""

  class Platform(Processor.Platform):
    def __init__(self, debug, wind_base):
      super(CRIOProcessor.Platform, self).__init__()

      self.__debug = debug
      self.__wind_base = wind_base

    def __repr__(self):
      return 'CRIOProcessor.Platform(debug=%s)' % self.debug()
    def __str__(self):
      return 'crio%s' % ('-debug' if self.debug() else '')

    def outname(self):
      return 'crio-debug' if self.debug() else 'crio'
    def os(self):
      return 'vxworks'
    def gyp_platform(self):
      return 'crio'
    def architecture(self):
      return 'ppc'
    def compiler(self):
      return 'gcc'
    def sanitizer(self):
      return 'none'
    def debug(self):
      return self.__debug
    def wind_base(self):
      return self.__wind_base

    # TODO(brians): test this
    def deploy(self, dry_run):
      self.do_deploy(dry_run,
                     ('ncftpput', get_ip('robot'), '/',
                      os.path.join(self.outdir(), 'lib',
                                   'FRC_UserProgram.out')))

    def build_env(self):
      return {'WIND_BASE': self.wind_base()}

  def __init__(self):
    super(CRIOProcessor, self).__init__()

    if 'WIND_BASE' in os.environ:
      self.__wind_base = os.environ['WIND_BASE']
    else:
      self.__wind_base = '/usr/local/powerpc-wrs-vxworks/wind_base'

  def parse_platforms(self, platforms_string):
    if platforms_string is None or platforms_string == 'crio':
      return (CRIOProcessor.Platform(False, self.wind_base()),)
    elif string == 'crio-debug' or string == 'debug':
      return (CRIOProcessor.Platform(True, self.wind_base()),)
    else:
      raise Processor.UnknownPlatform(
          'Unknown cRIO platform "%s".' % platforms_string)

  def wind_base(self):
    return self.__wind_base

  def extra_gyp_flags(self):
    return ('-DWIND_BASE=%s' % self.wind_base(),)

  def modify_ninja_file(self, ninja_file):
    subprocess.check_call(
        ('sed', '-i',
         's/nm -gD/nm/g', ninja_file),
        stdin=open(os.devnull, 'r'))

  def download_externals(self, _):
    call_download_externals('crio')

  def check_installed(self, platforms, is_deploy):
    packages = ('powerpc-wrs-vxworks', 'tcl')
    if is_deploy:
      packages += ('ncftp',)
    self.do_check_installed(packages)

class PrimeProcessor(Processor):
  """A Processor subclass for building prime code."""

  class Platform(Processor.Platform):
    def __init__(self, architecture, compiler, debug, sanitizer):
      super(PrimeProcessor.Platform, self).__init__()

      self.__architecture = architecture
      self.__compiler = compiler
      self.__debug = debug
      self.__sanitizer = sanitizer

    def __repr__(self):
      return 'PrimeProcessor.Platform(architecture=%s, compiler=%s, debug=%s' \
          ', sanitizer=%s)' \
          % (self.architecture(), self.compiler(), self.debug(),
             self.sanitizer())
    def __str__(self):
      return '%s-%s%s-%s' % (self.architecture(), self.compiler(),
                             '-debug' if self.debug() else '', self.sanitizer())

    def os(self):
      return 'linux'
    def gyp_platform(self):
      return '%s-%s-%s' % (self.os(), self.architecture(), self.compiler())
    def architecture(self):
      return self.__architecture
    def compiler(self):
      return self.__compiler
    def sanitizer(self):
      return self.__sanitizer
    def debug(self):
      return self.__debug

    def outname(self):
      return str(self)

    def priority(self):
      r = 0
      if self.compiler() == 'gcc':
        r -= 100
      elif self.compiler() == 'clang':
        r += 100
      if self.sanitizer() != 'none':
        r -= 50
      elif self.debug():
        r -= 10
      if self.architecture() == 'amd64':
        r += 5
      return r

    # TODO(brians): test this
    def deploy(self, dry_run):
      # Downloads code to the prime in a way that avoids clashing too badly with
      # starter (like the naive download everything one at a time).
      SUM = 'md5sum'
      TARGET_DIR = '/home/driver/robot_code/bin'
      TEMP_DIR = '/tmp/aos_downloader'
      TARGET = 'driver@' + get_ip('prime')

      from_dir = os.path.join(self.outdir(), 'outputs')
      sums = subprocess.check_output((SUM,) + tuple(os.listdir(from_dir)),
                                     stdin=open(os.devnull, 'r'),
                                     cwd=from_dir)
      to_download = subprocess.check_output(
          ('ssh', TARGET,
           """rm -rf {TMPDIR} && mkdir {TMPDIR} && cd {TO_DIR}
             && echo '{SUMS}' | {SUM} --check --quiet
             |& grep -F FAILED | sed 's/^\\(.*\\): FAILED.*"'$'"/\\1/g'""".
           format(TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR, SUMS=sums, SUM=SUM)))
      if not to_download:
        user_output("Nothing to download")
        return
      self.do_deploy(
          dry_run,
          ('scp', '-o', 'Compression yes') + to_download
          + (('%s:%s' % (TARGET, TEMP_DIR)),))
      if not dry_run:
        subprocess.check_call(
            ('ssh', TARGET,
             """mv {TMPDIR}/* {TO_DIR}
             && echo 'Done moving new executables into place'
             && ionice -c 3 bash -c 'sync && sync && sync'""".format(
                 TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR)))

    def build_env(self):
      OTHER_SYSROOT = '/opt/clang-3.5/'
      SYMBOLIZER_PATH = OTHER_SYSROOT + 'bin/llvm-symbolizer'
      r = {}
      if self.compiler() == 'clang' or self.compiler() == 'gcc_4.8':
        r['LD_LIBRARY_PATH'] = OTHER_SYSROOT + 'lib64'
      if self.sanitizer() == 'address':
        r['ASAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH
        r['ASAN_OPTIONS'] = \
            'detect_leaks=1:check_initialization_order=1:strict_init_order=1' \
            ':detect_stack_use_after_return=1:detect_odr_violation=2'
      elif self.sanitizer() == 'memory':
        r['MSAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH
      elif self.sanitizer() == 'thread':
        r['TSAN_OPTIONS'] = 'external_symbolizer_path=' + SYMBOLIZER_PATH

      r['CCACHE_COMPRESS'] = 'yes'
      r['CCACHE_DIR'] = os.path.abspath(os.path.join(aos_path(), '..', 'output',
                                                     'ccache_dir'))
      r['CCACHE_HASHDIR'] = 'yes'
      if self.compiler() == 'clang':
        # clang doesn't like being run directly on the preprocessed files.
        r['CCACHE_CPP2'] = 'yes'
      # Without this, ccache slows down because of the generated header files.
      # The race condition that this opens up isn't a problem because the build
      # system finishes modifying header files before compiling anything that
      # uses them.
      r['CCACHE_SLOPPINESS'] = 'include_file_mtime'

      if self.architecture() == 'amd64':
        r['PATH'] = os.path.join(aos_path(), 'build', 'bin-ld.gold') + \
            ':' + os.environ['PATH']

      return r

  ARCHITECTURES = ('arm', 'amd64')
  COMPILERS = ('clang', 'gcc', 'gcc_4.8')
  SANITIZERS = ('address', 'undefined', 'integer', 'memory', 'thread', 'none')
  SANITIZER_TEST_WARNINGS = {
      'memory': (True,
"""We don't have all of the libraries instrumented which leads to lots of false
  errors with msan (especially stdlibc++).
  TODO(brians): Figure out a way to deal with it."""),
  }
  PIE_SANITIZERS = ('memory', 'thread')

  def __init__(self, is_test, is_deploy):
    super(PrimeProcessor, self).__init__()

    platforms = []
    for architecture in PrimeProcessor.ARCHITECTURES:
      for compiler in PrimeProcessor.COMPILERS:
        for debug in [True, False]:
          if architecture == 'arm' and compiler == 'gcc_4.8':
            # We don't have a compiler to use here.
            continue
          platforms.append(
              PrimeProcessor.Platform(architecture, compiler, debug, 'none'))
    for sanitizer in PrimeProcessor.SANITIZERS:
      for compiler in ('gcc_4.8', 'clang'):
        if compiler == 'gcc_4.8' and (sanitizer == 'undefined' or
                                      sanitizer == 'integer' or
                                      sanitizer == 'memory'):
          # GCC 4.8 doesn't support these sanitizers.
          continue
        if sanitizer == 'none':
          # We already added sanitizer == 'none' above.
          continue
        platforms.append(
            PrimeProcessor.Platform('amd64', compiler, True, sanitizer))
    self.__platforms = frozenset(platforms)

    if is_test:
      default_platforms = self.select_platforms(architecture='amd64',
                                                debug=True)
      for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items():
        if warning[0]:
          default_platforms -= self.select_platforms(sanitizer=sanitizer)
    elif is_deploy:
      # TODO(brians): Switch to deploying the code built with clang.
      default_platforms = self.select_platforms(architecture='arm',
                                                compiler='gcc',
                                                debug=False)
    else:
      default_platforms = self.select_platforms(debug=False)
    self.__default_platforms = frozenset(default_platforms)

  def platforms(self):
    return self.__platforms
  def default_platforms(self):
    return self.__default_platforms

  def download_externals(self, platforms):
    to_download = set()
    for architecture in PrimeProcessor.ARCHITECTURES:
      for sanitizer in PrimeProcessor.PIE_SANITIZERS:
        if platforms & self.select_platforms(architecture=architecture,
                                             sanitizer=sanitizer):
          to_download.add(architecture + '-fPIE')
        if platforms & self.select_platforms(architecture=architecture,
                                             sanitizer='none'):
          to_download.add(architecture)
    for download_target in to_download:
      call_download_externals(download_target)

  def parse_platforms(self, platform_string):
    if platform_string is None:
      return self.default_platforms()
    r = self.default_platforms()
    for part in platform_string.split(','):
      if part == 'all':
        r = self.platforms()
      elif part[0] == '+':
        r = r | self.select_platforms_string(part[1:])
      elif part[0] == '-':
        r = r - self.select_platforms_string(part[1:])
      elif part[0] == '=':
        r = self.select_platforms_string(part[1:])
      else:
        selected = self.select_platforms_string(part)
        r = r - (self.platforms() - selected)
        if not r:
          r = selected
    return r

  def select_platforms(self, architecture=None, compiler=None, debug=None,
                       sanitizer=None):
    r = []
    for platform in self.platforms():
      if architecture is None or platform.architecture() == architecture:
        if compiler is None or platform.compiler() == compiler:
          if debug is None or platform.debug() == debug:
            if sanitizer is None or platform.sanitizer() == sanitizer:
              r.append(platform)
    return set(r)

  def select_platforms_string(self, platforms_string):
    architecture, compiler, debug, sanitizer = None, None, None, None
    for part in platforms_string.split('-'):
      if part in PrimeProcessor.ARCHITECTURES:
        architecture = part
      elif part in PrimeProcessor.COMPILERS:
        compiler = part
      elif part in ['debug', 'dbg']:
        debug = True
      elif part in ['release', 'nodebug', 'ndb']:
        debug = False
      elif part in PrimeProcessor.SANITIZERS:
        sanitizer = part
      else:
        raise Processor.UnknownPlatform(
            'Unknown platform string component "%s".' % part)
    return self.select_platforms(
        architecture=architecture,
        compiler=compiler,
        debug=debug,
        sanitizer=sanitizer)

  def check_installed(self, platforms, is_deploy):
    packages = set(('lzip', 'm4', 'realpath'))
    packages.add('ruby')
    # clang-format from here gets used for all versions.
    packages.add('clang-3.5')
    packages.add('arm-eabi-gcc')
    for platform in platforms:
      if platform.architecture() == 'arm':
        packages.add('gcc-4.7-arm-linux-gnueabihf')
        packages.add('g++-4.7-arm-linux-gnueabihf')
      if platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8':
        packages.add('clang-3.5')
      if platform.compiler() == 'gcc_4.8':
        packages.add('libcloog-isl3:amd64')
      if is_deploy:
        packages.add('openssh-client')
      if platform.compiler() == 'gcc' and platform.architecture() == 'amd64':
        packages.add('gcc-4.7')
        packages.add('g++-4.7')

    self.do_check_installed(tuple(packages))

def strsignal(num):
  # It ends up with SIGIOT instead otherwise, which is weird.
  if num == signal.SIGABRT:
    return 'SIGABRT'
  # SIGCLD is a weird way to spell it.
  if num == signal.SIGCHLD:
    return 'SIGCHLD'

  SIGNALS_TO_NAMES = dict((getattr(signal, n), n)
                          for n in dir(signal) if n.startswith('SIG')
                          and '_' not in n)
  return SIGNALS_TO_NAMES.get(num, 'Unknown signal %d' % num)

def main():
  class TryParsingAgain(Exception):
    pass

  class TryAgainArgumentParser(argparse.ArgumentParser):
    def __init__(self, **kwargs):
      super(TryAgainArgumentParser, self).__init__(**kwargs)

    def error(self, message):
      raise TryParsingAgain

  def set_up_parser(parser, args):
    def add_build_args(parser):
      parser.add_argument(
          'target',
          help='target to build',
          nargs='*')
      parser.add_argument(
          '--jobs', '-j',
          help='number of things to do at once',
          type=int)
    def add_common_args(parser):
      parser.add_argument(
          'platforms',
          help='platform(s) to act on',
          nargs='?')

    parser.add_argument('--processor', required=True, help='prime or crio')
    parser.add_argument('--main_gyp', required=True, help='main .gyp file')
    subparsers = parser.add_subparsers(dest='action_name')

    build_parser = subparsers.add_parser(
        'build',
        help='build the code (default)')
    add_common_args(build_parser)
    add_build_args(build_parser)

    clean_parser = subparsers.add_parser(
        'clean',
        help='remove all output directories')
    add_common_args(clean_parser)

    deploy_parser = subparsers.add_parser(
        'deploy',
        help='build and download the code')
    add_common_args(deploy_parser)
    add_build_args(deploy_parser)
    deploy_parser.add_argument(
        '-n', '--dry-run',
        help="don't actually download anything",
        action='store_true')

    tests_parser = subparsers.add_parser(
        'tests',
        help='run tests')
    add_common_args(tests_parser)
    add_build_args(tests_parser)

    return parser.parse_args(args)

  try:
    parser = TryAgainArgumentParser()
    args = set_up_parser(parser, sys.argv[1:])
  except TryParsingAgain:
    parser = argparse.ArgumentParser()
    REQUIRED_ARGS_END = 5
    args = set_up_parser(parser, sys.argv[1:REQUIRED_ARGS_END] + ['build'] +
                         sys.argv[(REQUIRED_ARGS_END):])

  if args.processor == 'crio':
    processor = CRIOProcessor()
  elif args.processor == 'prime':
    processor = PrimeProcessor(args.action_name == 'tests',
                               args.action_name == 'deploy')
  else:
    parser.exit(status=1, message='Unknown processor "%s".' % args.processor)

  if 'target' in args:
    targets = args.target[:]
  else:
    targets = []
  unknown_platform_error = None
  try:
    platforms = processor.parse_platforms(args.platforms)
  except Processor.UnknownPlatform as e:
    unknown_platform_error = e.message
    targets.append(args.platforms)
    platforms = processor.parse_platforms(None)
  if not platforms:
    user_output("No platforms selected!")
    exit(1)

  processor.check_installed(platforms, args.action_name == 'deploy')
  processor.download_externals(platforms)

  class ToolsConfig(object):
    def __init__(self):
      self.variables = {'AOS': aos_path()}
      with open(os.path.join(aos_path(), 'build', 'tools_config'), 'r') as f:
        for line in f:
          if line[0] == '#':
            pass
          elif line.isspace():
            pass
          else:
            new_name, new_value = line.rstrip().split('=')
            for name, value in self.variables.items():
              new_value = new_value.replace('${%s}' % name, value)
            self.variables[new_name] = new_value
    def __getitem__(self, key):
      return self.variables[key]

  tools_config = ToolsConfig()

  def handle_clean_error(function, path, excinfo):
    _, _ = function, path
    if issubclass(OSError, excinfo[0]):
      if excinfo[1].errno == errno.ENOENT:
        # Who cares if the file we're deleting isn't there?
        return
    raise excinfo[1]

  def need_to_run_gyp(platform):
    """Determines if we need to run gyp again or not.

    The generated build files are supposed to re-run gyp again themselves, but
    that doesn't work (or at least it used to not) and we sometimes want to
    modify the results anyways.

    Args:
      platform: The platform to check for.
    """
    if not os.path.exists(platform.build_ninja()):
      return True
    if os.path.getmtime(__file__) > os.path.getmtime(platform.build_ninja()):
      return True
    dirs = os.listdir(os.path.join(aos_path(), '..'))
    # Looking through these folders takes a long time and isn't useful.
    if dirs.count('output'):
      dirs.remove('output')
    if dirs.count('.git'):
      dirs.remove('.git')
    return not not subprocess.check_output(
        ('find',) + tuple(os.path.join(aos_path(), '..', d) for d in dirs)
        + ('-newer', platform.build_ninja(),
           '(', '-name', '*.gyp', '-or', '-name', '*.gypi', ')'),
        stdin=open(os.devnull, 'r'))

  def env(platform):
    """Makes sure we pass through important environmental variables.

    Returns:
      An environment suitable for passing to subprocess.Popen and friends.
    """
    build_env = dict(platform.build_env())
    if not 'TERM' in build_env:
      build_env['TERM'] = os.environ['TERM']
    if not 'PATH' in build_env:
      build_env['PATH'] = os.environ['PATH']
    return build_env

  sorted_platforms = sorted(platforms,
                            key=lambda platform: -platform.priority())

  to_build = []
  for platform in sorted_platforms:
    to_build.append(str(platform))
  if len(to_build) > 1:
    to_build[-1] = 'and ' + to_build[-1]
  user_output('Building %s...' % ', '.join(to_build))

  if args.action_name == 'tests':
    for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items():
      warned_about = platforms & processor.select_platforms(sanitizer=sanitizer)
      if warned_about:
        user_output(warning[1])
        if warning[0]:
          # TODO(brians): Add a --force flag or something to override this?
          user_output('Refusing to run tests for sanitizer %s.' % sanitizer)
          exit(1)

  num = 1
  for platform in sorted_platforms:
    user_output('Building %s (%d/%d)...' % (platform, num, len(platforms)))
    if args.action_name == 'clean':
      shutil.rmtree(platform.outdir(), onerror=handle_clean_error)
    else:
      if need_to_run_gyp(platform):
        user_output('Running gyp...')
        gyp = subprocess.Popen(
            (tools_config['GYP'],
             '--check',
             '--depth=%s' % os.path.join(aos_path(), '..'),
             '--no-circular-check',
             '-f', 'ninja',
             '-I%s' % os.path.join(aos_path(), 'build', 'aos.gypi'),
             '-I/dev/stdin', '-Goutput_dir=output',
             '-DOS=%s' % platform.os(),
             '-DPLATFORM=%s' % platform.gyp_platform(),
             '-DARCHITECTURE=%s' % platform.architecture(),
             '-DCOMPILER=%s' % platform.compiler().split('_')[0],
             '-DFULL_COMPILER=%s' % platform.compiler(),
             '-DDEBUG=%s' % ('yes' if platform.debug() else 'no'),
             '-DSANITIZER=%s' % platform.sanitizer(),
             '-DSANITIZER_FPIE=%s' %
             ('-fPIE' if platform.sanitizer() in PrimeProcessor.PIE_SANITIZERS
              else '')) +
            processor.extra_gyp_flags() + (args.main_gyp,),
            stdin=subprocess.PIPE)
        gyp.communicate(("""
{
  'target_defaults': {
    'configurations': {
      '%s': {}
    }
  }
}""" % platform.outname()).encode())
        if gyp.returncode:
          user_output("Running gyp failed!")
          exit(1)
        processor.modify_ninja_file(platform.build_ninja())
        user_output('Done running gyp')
      else:
        user_output("Not running gyp")

      try:
        call = (tools_config['NINJA'],
                '-C', platform.outdir()) + tuple(targets)
        if args.jobs:
          call += ('-j', str(args.jobs))
        subprocess.check_call(call,
                              stdin=open(os.devnull, 'r'),
                              env=env(platform))
      except subprocess.CalledProcessError as e:
        if unknown_platform_error is not None:
          user_output(unknown_platform_error)
        raise e

    if args.action_name == 'deploy':
      platform.deploy(args.dry_run)
    elif args.action_name == 'tests':
      dirname = os.path.join(platform.outdir(), 'tests')
      done_queue = queue.Queue()
      running = []
      if args.jobs:
        number_jobs = args.jobs
      else:
        number_jobs = os.sysconf('SC_NPROCESSORS_ONLN') + 2
      test_start_semaphore = threading.Semaphore(number_jobs)
      if targets:
        to_run = []
        for target in targets:
          if target.endswith('_test'):
            to_run.append(target)
      else:
        to_run = os.listdir(dirname)
      for f in to_run:
        thread = TestThread(os.path.join(dirname, f), env(platform), done_queue,
                            test_start_semaphore)
        running.append(thread)
        thread.start()
      try:
        while running:
          done = done_queue.get()
          running.remove(done)
          with test_output_lock:
            test_output('Output from test %s:' % done.name)
            with os.fdopen(done.output) as output_file:
              try:
                for line in output_file:
                  if not sys.stdout.isatty():
                    # Remove color escape codes.
                    line = re.sub(r'\x1B\[[0-9;]*[a-zA-Z]', '', line)
                  sys.stdout.write(line)
              except IOError as e:
# We want to just ignore EIOs from reading the master pty because that just
# means we hit the end.
                if e.errno != errno.EIO:
                  raise e
              if not done.returncode:
                test_output('Test %s succeeded' % done.name)
              else:
                if done.returncode < 0:
                  sig = -done.returncode
                  test_output('Test %s was killed by signal %d (%s)' % \
                              (done.name, sig, strsignal(sig)))
                elif done.returncode != 1:
                  test_output('Test %s exited with %d' % \
                              (done.name, done.returncode))
                else:
                  test_output('Test %s failed' % done.name)
                user_output('Aborting because of test failure for %s.' % \
                            platform)
                exit(1)
      finally:
        if running:
          test_output('Killing other tests...')
# Stop all of them before killing processes because otherwise stopping some of
# them tends to let other ones that are waiting to start go.
          for thread in running:
            test_output('\tKilling %s' % thread.name)
            thread.stop()
          for thread in running:
            thread.terminate_process()
          to_remove = []
          for thread in running:
            thread.join(5)
            if not thread.is_alive():
              to_remove.append(thread)
          for thread in to_remove:
            running.remove(thread)
          for thread in running:
            test_output(
                'Test %s did not terminate. Killing it.' % thread.name)
            thread.kill_process()
            thread.join()
          test_output('Done killing other tests')

    user_output('Done building %s (%d/%d)' % (platform, num, len(platforms)))
    num += 1

if __name__ == '__main__':
  main()