| #!/usr/bin/python3 |
| |
| import argparse |
| import sys |
| import subprocess |
| import re |
| import os |
| import os.path |
| import string |
| import shutil |
| import errno |
| import queue |
| import threading |
| |
| class TestThread(threading.Thread): |
| """Runs 1 test and keeps track of its current state. |
| |
| A TestThread is either waiting to start the test, actually running it, done, |
| running it, or stopped. The first 3 always happen in that order and can |
| change to stopped at any time. |
| |
| It will finish (ie join() will return) once the process has exited, at which |
| point accessing process to see the status is OK. |
| |
| Attributes: |
| executable: The file path of the executable to run. |
| env: The environment variables to set. |
| done_queue: A queue.Queue to place self on once done running the test. |
| start_semaphore: A threading.Semaphore to wait on before starting. |
| process_lock: A lock around process. |
| process: The currently executing test process or None. Synchronized by |
| process_lock. |
| stopped: True if we're stopped. |
| """ |
| def __init__(self, executable, env, done_queue, start_semaphore): |
| super(TestThread, self).__init__( |
| name=os.path.split(executable)[1]) |
| |
| self.executable = executable |
| self.env = env |
| self.done_queue = done_queue |
| self.start_semaphore = start_semaphore |
| |
| self.process_lock = threading.Lock() |
| self.process = None |
| self.stopped = False |
| self.returncode = None |
| self.output = None |
| |
| def run(self): |
| with self.start_semaphore: |
| if self.stopped: |
| return |
| test_output('Starting test %s...' % self.name) |
| self.output, subprocess_output = os.pipe() |
| with self.process_lock: |
| self.process = subprocess.Popen((self.executable, |
| '--gtest_color=yes'), |
| env=self.env, |
| stderr=subprocess.STDOUT, |
| stdout=subprocess_output, |
| stdin=open(os.devnull, 'r')) |
| os.close(subprocess_output) |
| self.process.wait() |
| with self.process_lock: |
| self.returncode = self.process.returncode |
| self.process = None |
| if not self.stopped: |
| self.done_queue.put(self) |
| |
| def terminate_process(self): |
| """Asks any currently running process to stop. |
| |
| Also changes this object to the stopped state. |
| """ |
| with self.process_lock: |
| self.stopped = True |
| if not self.process: |
| return |
| self.process.terminate() |
| def kill_process(self): |
| """Forcibly terminates any running process. |
| |
| Also changes this object to the stopped state. |
| """ |
| with self.process_lock: |
| self.stopped = True |
| if not self.process: |
| return |
| self.process.kill() |
| |
| def aos_path(): |
| """Returns: |
| A relative path to the aos directory. |
| """ |
| return os.path.join(os.path.dirname(__file__), '..') |
| |
| def get_ip(device): |
| """Retrieves the IP address for a given device.""" |
| FILENAME = os.path.normpath(os.path.join(aos_path(), '..', |
| 'output', 'ip_base.txt')) |
| if not os.access(FILENAME, os.R_OK): |
| os.makedirs(os.path.dirname(FILENAME), exist_ok=True) |
| with open(FILENAME, 'w') as f: |
| f.write('10.9.71') |
| with open(FILENAME, 'r') as f: |
| base = f.readline() |
| if device == 'prime': |
| return base + '.179' |
| elif device == 'robot': |
| return base + '.2' |
| else: |
| raise Exception('Unknown device %s to get an IP address for.' % device) |
| |
| def user_output(message): |
| """Prints message to the user.""" |
| print('build.py: ' + message, file=sys.stderr) |
| |
| # A lock to avoid making a mess intermingling test-related messages. |
| test_output_lock = threading.RLock() |
| def test_output(message): |
| """Prints message to the user. Intended for messages related to tests.""" |
| with test_output_lock: |
| print('tests: ' + message, file=sys.stdout) |
| |
| def call_download_externals(argument): |
| """Calls download_externals.sh for a given set of externals. |
| |
| Args: |
| argument: The argument to pass to the shell script to tell it what to |
| download. |
| """ |
| subprocess.check_call( |
| (os.path.join(aos_path(), 'build', 'download_externals.sh'), |
| argument), |
| stdin=open(os.devnull, 'r')) |
| |
| class Processor(object): |
| """Represents a processor architecture we can build for.""" |
| |
| class UnknownPlatform(Exception): |
| def __init__(self, message): |
| super(Processor.UnknownPlatform, self).__init__() |
| self.message = message |
| |
| class Platform(object): |
| """Represents a single way to build the code.""" |
| |
| def outdir(self): |
| """Returns: |
| The path of the directory build outputs get put in to. |
| """ |
| return os.path.join( |
| aos_path(), '..', 'output', self.outname()) |
| def build_ninja(self): |
| """Returns: |
| The path of the build.ninja file. |
| """ |
| return os.path.join(self.outdir(), 'build.ninja') |
| |
| def do_deploy(self, dry_run, command): |
| """Helper for subclasses to implement deploy. |
| |
| Args: |
| dry_run: If True, prints the command instead of actually running it. |
| command: A tuple of command-line arguments. |
| """ |
| real_command = (('echo',) + command) if dry_run else command |
| subprocess.check_call(real_command, stdin=open(os.devnull, 'r')) |
| |
| def deploy(self, dry_run): |
| """Downloads the compiled code to the target computer.""" |
| raise NotImplementedError('deploy should be overriden') |
| def outname(self): |
| """Returns: |
| The name of the directory the code will be compiled to. |
| """ |
| raise NotImplementedError('outname should be overriden') |
| def os(self): |
| """Returns: |
| The name of the operating system this platform is for. |
| |
| This will be used as the value of the OS gyp variable. |
| """ |
| raise NotImplementedError('os should be overriden') |
| def gyp_platform(self): |
| """Returns: |
| The platform name the .gyp files know. |
| |
| This will be used as the value of the PLATFORM gyp variable. |
| """ |
| raise NotImplementedError('gyp_platform should be overriden') |
| def architecture(self): |
| """Returns: |
| The processor architecture for this platform. |
| |
| This will be used as the value of the ARCHITECTURE gyp variable. |
| """ |
| raise NotImplementedError('architecture should be overriden') |
| def compiler(self): |
| """Returns: |
| The compiler used for this platform. |
| |
| Everything before the first _ will be used as the value of the |
| COMPILER gyp variable and the whole thing will be used as the value |
| of the FULL_COMPILER gyp variable. |
| """ |
| raise NotImplementedError('compiler should be overriden') |
| def sanitizer(self): |
| """Returns: |
| The sanitizer used on this platform. |
| |
| This will be used as the value of the SANITIZER gyp variable. |
| |
| "none" if there isn't one. |
| """ |
| raise NotImplementedError('sanitizer should be overriden') |
| def debug(self): |
| """Returns: |
| Whether or not this platform compiles with debugging information. |
| |
| The DEBUG gyp variable will be set to "yes" or "no" based on this. |
| """ |
| raise NotImplementedError('debug should be overriden') |
| def build_env(self): |
| """Returns: |
| A map of environment variables to set while building this platform. |
| """ |
| raise NotImplementedError('build_env should be overriden') |
| |
| def check_installed(self, platforms, is_deploy): |
| """Makes sure that everything necessary to build platforms are installed.""" |
| raise NotImplementedError('check_installed should be overriden') |
| def parse_platforms(self, platforms_string): |
| """Args: |
| string: A user-supplied string saying which platforms to select. |
| |
| Returns: |
| A tuple of Platform objects. |
| |
| Raises: |
| Processor.UnknownPlatform: Parsing string didn't work out. |
| """ |
| raise NotImplementedError('parse_platforms should be overriden') |
| def extra_gyp_flags(self): |
| """Returns: |
| A tuple of extra flags to pass to gyp (if any). |
| """ |
| return () |
| def modify_ninja_file(self, ninja_file): |
| """Modifies a freshly generated ninja file as necessary. |
| |
| Args: |
| ninja_file: Path to the file to modify. |
| """ |
| pass |
| def download_externals(self, platforms): |
| """Calls download_externals as appropriate to build platforms. |
| |
| Args: |
| platforms: A list of platforms to download external libraries for. |
| """ |
| raise NotImplementedError('download_externals should be overriden') |
| |
| def do_check_installed(self, other_packages): |
| """Helper for subclasses to implement check_installed. |
| |
| Args: |
| other_packages: A tuple of platform-specific packages to check for.""" |
| all_packages = other_packages |
| # Necessary to build stuff. |
| all_packages += ('ccache', 'make') |
| # Necessary to download stuff to build. |
| all_packages += ('wget', 'git', 'subversion', 'patch', 'unzip', 'bzip2') |
| # Necessary to build externals stuff. |
| all_packages += ('python', 'gcc', 'g++') |
| try: |
| # TODO(brians): Check versions too. |
| result = subprocess.check_output( |
| ('dpkg-query', '--show') + all_packages, |
| stdin=open(os.devnull, 'r'), |
| stderr=subprocess.STDOUT) |
| except subprocess.CalledProcessError as e: |
| output = e.output.decode('utf-8').rstrip() |
| not_found = [] |
| for line in output.splitlines(True): |
| match = re.match(r'dpkg-query: no packages found matching (.*)', |
| line) |
| if match: |
| not_found.append(match.group(1)) |
| user_output('Some packages not installed: %s.' % ', '.join(not_found)) |
| user_output('Try something like `sudo apt-get install %s`.' % |
| ' '.join(not_found)) |
| exit(1) |
| |
| class CRIOProcessor(Processor): |
| """A Processor subclass for building cRIO code.""" |
| |
| class Platform(Processor.Platform): |
| def __init__(self, debug, wind_base): |
| super(CRIOProcessor.Platform, self).__init__() |
| |
| self.__debug = debug |
| self.__wind_base = wind_base |
| |
| def __repr__(self): |
| return 'CRIOProcessor.Platform(debug=%s)' % self.debug() |
| def __str__(self): |
| return 'crio%s' % ('-debug' if self.debug() else '') |
| |
| def outname(self): |
| return 'crio-debug' if self.debug() else 'crio' |
| def os(self): |
| return 'vxworks' |
| def gyp_platform(self): |
| return 'crio' |
| def architecture(self): |
| return 'ppc' |
| def compiler(self): |
| return 'gcc' |
| def sanitizer(self): |
| return 'none' |
| def debug(self): |
| return self.__debug |
| def wind_base(self): |
| return self.__wind_base |
| |
| # TODO(brians): test this |
| def deploy(self, dry_run): |
| self.do_deploy(dry_run, |
| ('ncftpput', get_ip('robot'), '/', |
| os.path.join(self.outdir(), 'lib', |
| 'FRC_UserProgram.out'))) |
| |
| def build_env(self): |
| return {'WIND_BASE': self.wind_base()} |
| |
| def __init__(self): |
| super(CRIOProcessor, self).__init__() |
| |
| if 'WIND_BASE' in os.environ: |
| self.__wind_base = os.environ['WIND_BASE'] |
| else: |
| self.__wind_base = '/usr/local/powerpc-wrs-vxworks/wind_base' |
| |
| def parse_platforms(self, platforms_string): |
| if platforms_string is None or platforms_string == 'crio': |
| return (CRIOProcessor.Platform(False, self.wind_base()),) |
| elif string == 'crio-debug' or string == 'debug': |
| return (CRIOProcessor.Platform(True, self.wind_base()),) |
| else: |
| raise Processor.UnknownPlatform( |
| 'Unknown cRIO platform "%s".' % platforms_string) |
| |
| def wind_base(self): |
| return self.__wind_base |
| |
| def extra_gyp_flags(self): |
| return ('-DWIND_BASE=%s' % self.wind_base(),) |
| |
| def modify_ninja_file(self, ninja_file): |
| subprocess.check_call( |
| ('sed', '-i', |
| 's/nm -gD/nm/g', ninja_file), |
| stdin=open(os.devnull, 'r')) |
| |
| def download_externals(self, _): |
| call_download_externals('crio') |
| |
| def check_installed(self, platforms, is_deploy): |
| packages = ('powerpc-wrs-vxworks', 'tcl') |
| if is_deploy: |
| packages += ('ncftp',) |
| self.do_check_installed(packages) |
| |
| class PrimeProcessor(Processor): |
| """A Processor subclass for building prime code.""" |
| |
| class Platform(Processor.Platform): |
| def __init__(self, architecture, compiler, debug, sanitizer): |
| super(PrimeProcessor.Platform, self).__init__() |
| |
| self.__architecture = architecture |
| self.__compiler = compiler |
| self.__debug = debug |
| self.__sanitizer = sanitizer |
| |
| def __repr__(self): |
| return 'PrimeProcessor.Platform(architecture=%s, compiler=%s, debug=%s' \ |
| ', sanitizer=%s)' \ |
| % (self.architecture(), self.compiler(), self.debug(), |
| self.sanitizer()) |
| def __str__(self): |
| return '%s-%s%s-%s' % (self.architecture(), self.compiler(), |
| '-debug' if self.debug() else '', self.sanitizer()) |
| |
| def os(self): |
| return 'linux' |
| def gyp_platform(self): |
| return '%s-%s-%s' % (self.os(), self.architecture(), self.compiler()) |
| def architecture(self): |
| return self.__architecture |
| def compiler(self): |
| return self.__compiler |
| def sanitizer(self): |
| return self.__sanitizer |
| def debug(self): |
| return self.__debug |
| |
| def outname(self): |
| return str(self) |
| |
| # TODO(brians): test this |
| def deploy(self, dry_run): |
| # Downloads code to the prime in a way that avoids clashing too badly with |
| # starter (like the naive download everything one at a time). |
| SUM = 'md5sum' |
| TARGET_DIR = '/home/driver/robot_code/bin' |
| TEMP_DIR = '/tmp/aos_downloader' |
| TARGET = 'driver@' + get_ip('prime') |
| |
| from_dir = os.path.join(self.outdir(), 'outputs') |
| sums = subprocess.check_output((SUM,) + tuple(os.listdir(from_dir)), |
| stdin=open(os.devnull, 'r'), |
| cwd=from_dir) |
| to_download = subprocess.check_output( |
| ('ssh', TARGET, |
| """rm -rf {TMPDIR} && mkdir {TMPDIR} && cd {TO_DIR} |
| && echo '{SUMS}' | {SUM} --check --quiet |
| |& grep -F FAILED | sed 's/^\\(.*\\): FAILED.*"'$'"/\\1/g'""". |
| format(TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR, SUMS=sums, SUM=SUM))) |
| if not to_download: |
| user_output("Nothing to download") |
| return |
| self.do_deploy( |
| dry_run, |
| ('scp', '-o', 'Compression yes') + to_download |
| + (('%s:%s' % (TARGET, TEMP_DIR)),)) |
| if not dry_run: |
| subprocess.check_call( |
| ('ssh', TARGET, |
| """mv {TMPDIR}/* {TO_DIR} |
| && echo 'Done moving new executables into place' |
| && ionice -c 3 bash -c 'sync && sync && sync'""".format( |
| TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR))) |
| |
| def build_env(self): |
| OTHER_SYSROOT = '/opt/clang-3.5/' |
| SYMBOLIZER_PATH = OTHER_SYSROOT + 'bin/llvm-symbolizer' |
| r = {} |
| if self.compiler() == 'clang' or self.compiler() == 'gcc_4.8': |
| r['LD_LIBRARY_PATH'] = OTHER_SYSROOT + 'lib64' |
| if self.sanitizer() == 'address': |
| r['ASAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH |
| r['ASAN_OPTIONS'] = \ |
| 'detect_leaks=1:check_initialization_order=1:strict_init_order=1' |
| elif self.sanitizer() == 'memory': |
| r['MSAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH |
| elif self.sanitizer() == 'thread': |
| r['TSAN_OPTIONS'] = 'external_symbolizer_path=' + SYMBOLIZER_PATH |
| |
| r['CCACHE_COMPRESS'] = 'yes' |
| r['CCACHE_DIR'] = os.path.abspath(os.path.join(aos_path(), '..', 'output', |
| 'ccache_dir')) |
| r['CCACHE_HASHDIR'] = 'yes' |
| if self.compiler() == 'clang': |
| # clang doesn't like being run directly on the preprocessed files. |
| r['CCACHE_CPP2'] = 'yes' |
| # Without this, ccache slows down because of the generated header files. |
| # The race condition that this opens up isn't a problem because the build |
| # system finishes modifying header files before compiling anything that |
| # uses them. |
| r['CCACHE_SLOPPINESS'] = 'include_file_mtime' |
| |
| if self.architecture() == 'amd64': |
| r['PATH'] = os.path.join(aos_path(), 'build', 'bin-ld.gold') + \ |
| ':' + os.environ['PATH'] |
| |
| return r |
| |
| ARCHITECTURES = ('arm', 'amd64') |
| COMPILERS = ('clang', 'gcc', 'gcc_4.8') |
| SANITIZERS = ('address', 'undefined', 'integer', 'memory', 'thread', 'none') |
| SANITIZER_TEST_WARNINGS = { |
| 'memory': (True, |
| """We don't have all of the libraries instrumented which leads to lots of false |
| errors with msan (especially stdlibc++). |
| TODO(brians): Figure out a way to deal with it."""), |
| 'undefined': (False, |
| """There are several warnings in other people's code that ubsan catches. |
| The following have been verified non-interesting: |
| include/c++/4.8.2/array:*: runtime error: reference binding to null pointer |
| of type 'int' |
| This happens with ::std::array<T, 0> and it doesn't seem to cause any |
| issues. |
| output/downloaded/eigen-3.2.1/Eigen/src/Core/util/Memory.h:782:*: runtime |
| error: load of misaligned address 0x* for type 'const int', which |
| requires 4 byte alignment |
| That's in the CPUID detection code which only runs on x86."""), |
| } |
| PIE_SANITIZERS = ('memory', 'thread') |
| |
| def __init__(self, is_test, is_deploy): |
| super(PrimeProcessor, self).__init__() |
| |
| platforms = [] |
| for architecture in PrimeProcessor.ARCHITECTURES: |
| for compiler in PrimeProcessor.COMPILERS: |
| for debug in [True, False]: |
| if architecture == 'arm' and compiler == 'gcc_4.8': |
| # We don't have a compiler to use here. |
| continue |
| platforms.append( |
| PrimeProcessor.Platform(architecture, compiler, debug, 'none')) |
| for sanitizer in PrimeProcessor.SANITIZERS: |
| for compiler in ('gcc_4.8', 'clang'): |
| if compiler == 'gcc_4.8' and (sanitizer == 'undefined' or |
| sanitizer == 'integer' or |
| sanitizer == 'memory'): |
| # GCC 4.8 doesn't support these sanitizers. |
| continue |
| if sanitizer == 'none': |
| # We already added sanitizer == 'none' above. |
| continue |
| platforms.append( |
| PrimeProcessor.Platform('amd64', compiler, True, sanitizer)) |
| self.__platforms = frozenset(platforms) |
| |
| if is_test: |
| default_platforms = self.select_platforms(architecture='amd64', |
| debug=True) |
| for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items(): |
| if warning[0]: |
| default_platforms -= self.select_platforms(sanitizer=sanitizer) |
| elif is_deploy: |
| # TODO(brians): Switch to deploying the code built with clang. |
| default_platforms = self.select_platforms(architecture='arm', |
| compiler='gcc', |
| debug=False) |
| else: |
| default_platforms = self.select_platforms(debug=False) |
| self.__default_platforms = frozenset(default_platforms) |
| |
| def platforms(self): |
| return self.__platforms |
| def default_platforms(self): |
| return self.__default_platforms |
| |
| def download_externals(self, platforms): |
| to_download = set() |
| for architecture in PrimeProcessor.ARCHITECTURES: |
| for sanitizer in PrimeProcessor.PIE_SANITIZERS: |
| if platforms & self.select_platforms(architecture=architecture, |
| sanitizer=sanitizer): |
| to_download.add(architecture + '-fPIE') |
| if platforms & self.select_platforms(architecture=architecture, |
| sanitizer='none'): |
| to_download.add(architecture) |
| for download_target in to_download: |
| call_download_externals(download_target) |
| |
| def parse_platforms(self, platform_string): |
| if platform_string is None: |
| return self.default_platforms() |
| r = self.default_platforms() |
| for part in platform_string.split(','): |
| if part == 'all': |
| r = self.platforms() |
| elif part[0] == '+': |
| r = r | self.select_platforms_string(part[1:]) |
| elif part[0] == '-': |
| r = r - self.select_platforms_string(part[1:]) |
| elif part[0] == '=': |
| r = self.select_platforms_string(part[1:]) |
| else: |
| selected = self.select_platforms_string(part) |
| r = r - (self.platforms() - selected) |
| if not r: |
| r = selected |
| return r |
| |
| def select_platforms(self, architecture=None, compiler=None, debug=None, |
| sanitizer=None): |
| r = [] |
| for platform in self.platforms(): |
| if architecture is None or platform.architecture() == architecture: |
| if compiler is None or platform.compiler() == compiler: |
| if debug is None or platform.debug() == debug: |
| if sanitizer is None or platform.sanitizer() == sanitizer: |
| r.append(platform) |
| return set(r) |
| |
| def select_platforms_string(self, platforms_string): |
| architecture, compiler, debug, sanitizer = None, None, None, None |
| for part in platforms_string.split('-'): |
| if part in PrimeProcessor.ARCHITECTURES: |
| architecture = part |
| elif part in PrimeProcessor.COMPILERS: |
| compiler = part |
| elif part in ['debug', 'dbg']: |
| debug = True |
| elif part in ['release', 'nodebug', 'ndb']: |
| debug = False |
| elif part in PrimeProcessor.SANITIZERS: |
| sanitizer = part |
| else: |
| raise Processor.UnknownPlatform( |
| 'Unknown platform string component "%s".' % part) |
| return self.select_platforms( |
| architecture=architecture, |
| compiler=compiler, |
| debug=debug, |
| sanitizer=sanitizer) |
| |
| def check_installed(self, platforms, is_deploy): |
| packages = set(('lzip', 'm4', 'realpath')) |
| packages.add('ruby') |
| # clang-format from here gets used for all versions. |
| packages.add('clang-3.5') |
| packages.add('arm-eabi-gcc') |
| for platform in platforms: |
| if platform.architecture() == 'arm': |
| packages.add('gcc-4.7-arm-linux-gnueabihf') |
| packages.add('g++-4.7-arm-linux-gnueabihf') |
| if platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8': |
| packages.add('clang-3.5') |
| if platform.compiler() == 'gcc_4.8': |
| packages.add('libcloog-isl3:amd64') |
| if is_deploy: |
| packages.add('openssh-client') |
| if platform.compiler() == 'gcc' and platform.architecture() == 'amd64': |
| packages.add('gcc-4.7') |
| packages.add('g++-4.7') |
| |
| self.do_check_installed(tuple(packages)) |
| |
| def main(): |
| class TryParsingAgain(Exception): |
| pass |
| |
| class TryAgainArgumentParser(argparse.ArgumentParser): |
| def __init__(self, **kwargs): |
| super(TryAgainArgumentParser, self).__init__(**kwargs) |
| |
| def error(self, message): |
| raise TryParsingAgain |
| |
| def set_up_parser(parser, args): |
| def add_build_args(parser): |
| parser.add_argument( |
| 'target', |
| help='target to build', |
| nargs='*') |
| parser.add_argument( |
| '--jobs', '-j', |
| help='number of things to do at once', |
| type=int) |
| def add_common_args(parser): |
| parser.add_argument( |
| 'platforms', |
| help='platform(s) to act on', |
| nargs='?') |
| |
| parser.add_argument('--processor', required=True, help='prime or crio') |
| parser.add_argument('--main_gyp', required=True, help='main .gyp file') |
| subparsers = parser.add_subparsers(dest='action_name') |
| |
| build_parser = subparsers.add_parser( |
| 'build', |
| help='build the code (default)') |
| add_common_args(build_parser) |
| add_build_args(build_parser) |
| |
| clean_parser = subparsers.add_parser( |
| 'clean', |
| help='remove all output directories') |
| add_common_args(clean_parser) |
| |
| deploy_parser = subparsers.add_parser( |
| 'deploy', |
| help='build and download the code') |
| add_common_args(deploy_parser) |
| add_build_args(deploy_parser) |
| deploy_parser.add_argument( |
| '-n', '--dry-run', |
| help="don't actually download anything", |
| action='store_true') |
| |
| tests_parser = subparsers.add_parser( |
| 'tests', |
| help='run tests') |
| add_common_args(tests_parser) |
| add_build_args(tests_parser) |
| |
| return parser.parse_args(args) |
| |
| try: |
| parser = TryAgainArgumentParser() |
| args = set_up_parser(parser, sys.argv[1:]) |
| except TryParsingAgain: |
| parser = argparse.ArgumentParser() |
| REQUIRED_ARGS_END = 5 |
| args = set_up_parser(parser, sys.argv[1:REQUIRED_ARGS_END] + ['build'] + |
| sys.argv[(REQUIRED_ARGS_END):]) |
| |
| if args.processor == 'crio': |
| processor = CRIOProcessor() |
| elif args.processor == 'prime': |
| processor = PrimeProcessor(args.action_name == 'tests', |
| args.action_name == 'deploy') |
| else: |
| parser.exit(status=1, message='Unknown processor "%s".' % args.processor) |
| |
| if 'target' in args: |
| targets = args.target[:] |
| else: |
| targets = [] |
| unknown_platform_error = None |
| try: |
| platforms = processor.parse_platforms(args.platforms) |
| except Processor.UnknownPlatform as e: |
| unknown_platform_error = e.message |
| targets.append(args.platforms) |
| platforms = processor.parse_platforms(None) |
| if not platforms: |
| user_output("No platforms selected!") |
| exit(1) |
| |
| processor.check_installed(platforms, args.action_name == 'deploy') |
| processor.download_externals(platforms) |
| |
| class ToolsConfig(object): |
| def __init__(self): |
| self.variables = {'AOS': aos_path()} |
| with open(os.path.join(aos_path(), 'build', 'tools_config'), 'r') as f: |
| for line in f: |
| if line[0] == '#': |
| pass |
| elif line.isspace(): |
| pass |
| else: |
| new_name, new_value = line.rstrip().split('=') |
| for name, value in self.variables.items(): |
| new_value = new_value.replace('${%s}' % name, value) |
| self.variables[new_name] = new_value |
| def __getitem__(self, key): |
| return self.variables[key] |
| |
| tools_config = ToolsConfig() |
| |
| def handle_clean_error(function, path, excinfo): |
| _, _ = function, path |
| if issubclass(OSError, excinfo[0]): |
| if excinfo[1].errno == errno.ENOENT: |
| # Who cares if the file we're deleting isn't there? |
| return |
| raise excinfo[1] |
| |
| def need_to_run_gyp(platform): |
| """Determines if we need to run gyp again or not. |
| |
| The generated build files are supposed to re-run gyp again themselves, but |
| that doesn't work (or at least it used to not) and we sometimes want to |
| modify the results anyways. |
| |
| Args: |
| platform: The platform to check for. |
| """ |
| if not os.path.exists(platform.build_ninja()): |
| return True |
| if os.path.getmtime(__file__) > os.path.getmtime(platform.build_ninja()): |
| return True |
| dirs = os.listdir(os.path.join(aos_path(), '..')) |
| # Looking through these folders takes a long time and isn't useful. |
| if dirs.count('output'): |
| dirs.remove('output') |
| if dirs.count('.git'): |
| dirs.remove('.git') |
| return not not subprocess.check_output( |
| ('find',) + tuple(os.path.join(aos_path(), '..', d) for d in dirs) |
| + ('-newer', platform.build_ninja(), |
| '(', '-name', '*.gyp', '-or', '-name', '*.gypi', ')'), |
| stdin=open(os.devnull, 'r')) |
| |
| def env(platform): |
| """Makes sure we pass through important environmental variables. |
| |
| Returns: |
| An environment suitable for passing to subprocess.Popen and friends. |
| """ |
| build_env = dict(platform.build_env()) |
| if not 'TERM' in build_env: |
| build_env['TERM'] = os.environ['TERM'] |
| if not 'PATH' in build_env: |
| build_env['PATH'] = os.environ['PATH'] |
| return build_env |
| |
| to_build = [] |
| for platform in platforms: |
| to_build.append(str(platform)) |
| if len(to_build) > 1: |
| to_build[-1] = 'and ' + to_build[-1] |
| user_output('Building %s...' % ', '.join(to_build)) |
| |
| if args.action_name == 'tests': |
| for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items(): |
| warned_about = platforms & processor.select_platforms(sanitizer=sanitizer) |
| if warned_about: |
| user_output(warning[1]) |
| if warning[0]: |
| # TODO(brians): Add a --force flag or something to override this? |
| user_output('Refusing to run tests for sanitizer %s.' % sanitizer) |
| exit(1) |
| |
| num = 1 |
| for platform in platforms: |
| user_output('Building %s (%d/%d)...' % (platform, num, len(platforms))) |
| if args.action_name == 'clean': |
| shutil.rmtree(platform.outdir(), onerror=handle_clean_error) |
| else: |
| if need_to_run_gyp(platform): |
| user_output('Running gyp...') |
| gyp = subprocess.Popen( |
| (tools_config['GYP'], |
| '--check', |
| '--depth=%s' % os.path.join(aos_path(), '..'), |
| '--no-circular-check', |
| '-f', 'ninja', |
| '-I%s' % os.path.join(aos_path(), 'build', 'aos.gypi'), |
| '-I/dev/stdin', '-Goutput_dir=output', |
| '-DOS=%s' % platform.os(), |
| '-DPLATFORM=%s' % platform.gyp_platform(), |
| '-DARCHITECTURE=%s' % platform.architecture(), |
| '-DCOMPILER=%s' % platform.compiler().split('_')[0], |
| '-DFULL_COMPILER=%s' % platform.compiler(), |
| '-DDEBUG=%s' % ('yes' if platform.debug() else 'no'), |
| '-DSANITIZER=%s' % platform.sanitizer(), |
| '-DSANITIZER_FPIE=%s' % |
| ('-fPIE' if platform.sanitizer() in PrimeProcessor.PIE_SANITIZERS |
| else '')) + |
| processor.extra_gyp_flags() + (args.main_gyp,), |
| stdin=subprocess.PIPE) |
| gyp.communicate((""" |
| { |
| 'target_defaults': { |
| 'configurations': { |
| '%s': {} |
| } |
| } |
| }""" % platform.outname()).encode()) |
| if gyp.returncode: |
| user_output("Running gyp failed!") |
| exit(1) |
| processor.modify_ninja_file(platform.build_ninja()) |
| user_output('Done running gyp') |
| else: |
| user_output("Not running gyp") |
| |
| try: |
| call = (tools_config['NINJA'], |
| '-C', platform.outdir()) + tuple(targets) |
| if args.jobs: |
| call += ('-j', str(args.jobs)) |
| subprocess.check_call(call, |
| stdin=open(os.devnull, 'r'), |
| env=env(platform)) |
| except subprocess.CalledProcessError as e: |
| if unknown_platform_error is not None: |
| user_output(unknown_platform_error) |
| raise e |
| |
| if args.action_name == 'deploy': |
| platform.deploy(args.dry_run) |
| elif args.action_name == 'tests': |
| dirname = os.path.join(platform.outdir(), 'tests') |
| done_queue = queue.Queue() |
| running = [] |
| if args.jobs: |
| number_jobs = args.jobs |
| else: |
| number_jobs = os.sysconf('SC_NPROCESSORS_ONLN') + 2 |
| test_start_semaphore = threading.Semaphore(number_jobs) |
| if targets: |
| to_run = [] |
| for target in targets: |
| if target.endswith('_test'): |
| to_run.append(target) |
| else: |
| to_run = os.listdir(dirname) |
| for f in to_run: |
| thread = TestThread(os.path.join(dirname, f), env(platform), done_queue, |
| test_start_semaphore) |
| running.append(thread) |
| thread.start() |
| try: |
| while running: |
| done = done_queue.get() |
| running.remove(done) |
| with test_output_lock: |
| test_output('Output from test %s:' % done.name) |
| for line in os.fdopen(done.output): |
| if not sys.stdout.isatty(): |
| # Remove color escape codes. |
| line = re.sub(r'\x1B\[[0-9;]*[a-zA-Z]', '', line) |
| sys.stdout.write(line) |
| if not done.returncode: |
| test_output('Test %s succeeded' % done.name) |
| else: |
| test_output('Test %s failed' % done.name) |
| user_output('Aborting because of test failure.') |
| exit(1) |
| finally: |
| if running: |
| test_output('Killing other tests...') |
| for thread in running: |
| thread.terminate_process() |
| to_remove = [] |
| for thread in running: |
| thread.join(5) |
| if not thread.is_alive(): |
| to_remove.append(thread) |
| for thread in to_remove: |
| running.remove(thread) |
| for thread in running: |
| test_output( |
| 'Test %s did not terminate. Killing it.' % thread.name) |
| thread.kill_process() |
| thread.join() |
| test_output('Done killing other tests') |
| |
| user_output('Done building %s (%d/%d)' % (platform, num, len(platforms))) |
| num += 1 |
| |
| if __name__ == '__main__': |
| main() |