| #!/usr/bin/python3 |
| |
| import sys |
| import subprocess |
| import re |
| import os |
| import os.path |
| import string |
| import shutil |
| import errno |
| import queue |
| import threading |
| import pty |
| import signal |
| |
| class TestThread(threading.Thread): |
| """Runs 1 test and keeps track of its current state. |
| |
| A TestThread is either waiting to start the test, actually running it, done, |
| running it, or stopped. The first 3 always happen in that order and can |
| change to stopped at any time. |
| |
| It will finish (ie join() will return) once the process has exited, at which |
| point accessing process to see the status is OK. |
| |
| Attributes: |
| executable: The file path of the executable to run. |
| args: A tuple of arguments to give the executable. |
| env: The environment variables to set. |
| done_queue: A queue.Queue to place self on once done running the test. |
| start_semaphore: A threading.Semaphore to wait on before starting. |
| process_lock: A lock around process. |
| process: The currently executing test process or None. Synchronized by |
| process_lock. |
| stopped: True if we're stopped. |
| output: A queue of lines of output from the test. |
| """ |
| |
| class OutputCopier(threading.Thread): |
| """Copies the output of a test from its output pty into a queue. |
| |
| This is necessary because otherwise everything locks up if the test writes |
| too much output and fills up the pty's buffer. |
| """ |
| |
| def __init__(self, name, fd, queue): |
| super(TestThread.OutputCopier, self).__init__( |
| name=(name + '.OutputCopier')) |
| |
| self.fd = fd |
| self.queue = queue |
| |
| def run(self): |
| with os.fdopen(self.fd) as to_read: |
| try: |
| for line in to_read: |
| self.queue.put(line) |
| except IOError as e: |
| # An EIO from the master side of the pty means we hit the end. |
| if e.errno == errno.EIO: |
| return |
| else: |
| raise e |
| |
| def __init__(self, executable, args, env, done_queue, start_semaphore): |
| super(TestThread, self).__init__( |
| name=os.path.split(executable)[-1]) |
| |
| self.executable = executable |
| self.args = args |
| self.env = env |
| self.done_queue = done_queue |
| self.start_semaphore = start_semaphore |
| |
| self.output = queue.Queue() |
| |
| self.process_lock = threading.Lock() |
| self.process = None |
| self.stopped = False |
| self.returncode = None |
| self.output_copier = None |
| |
| def run(self): |
| def setup_test_process(): |
| # Shove it into its own process group so we can kill any subprocesses easily. |
| os.setpgid(0, 0) |
| |
| with self.start_semaphore: |
| with self.process_lock: |
| if self.stopped: |
| return |
| test_output('Starting test %s...' % self.name) |
| output_to_read, subprocess_output = pty.openpty() |
| self.output_copier = TestThread.OutputCopier(self.name, output_to_read, |
| self.output) |
| self.output_copier.start() |
| try: |
| with self.process_lock: |
| self.process = subprocess.Popen((self.name,) + self.args, |
| executable=self.executable, |
| env=self.env, |
| stderr=subprocess.STDOUT, |
| stdout=subprocess_output, |
| stdin=open(os.devnull, 'r'), |
| preexec_fn=setup_test_process) |
| finally: |
| os.close(subprocess_output) |
| self.process.wait() |
| with self.process_lock: |
| self.returncode = self.process.returncode |
| self.process = None |
| if not self.stopped: |
| self.output_copier.join() |
| self.done_queue.put(self) |
| |
| def kill_process(self): |
| """Forcibly terminates any running process.""" |
| with self.process_lock: |
| if not self.process: |
| return |
| try: |
| os.killpg(self.process.pid, signal.SIGKILL) |
| except OSError as e: |
| if e.errno == errno.ESRCH: |
| # We don't really care if it's already gone. |
| pass |
| else: |
| raise e |
| def stop(self): |
| """Changes self to the stopped state.""" |
| with self.process_lock: |
| self.stopped = True |
| |
| def aos_path(): |
| """Returns: |
| A relative path to the aos directory. |
| """ |
| return os.path.join(os.path.dirname(__file__), '..') |
| |
| def get_ip(): |
| """Retrieves the IP address to download code to.""" |
| FILENAME = os.path.normpath(os.path.join(aos_path(), '..', |
| 'output', 'ip_address.txt')) |
| if not os.access(FILENAME, os.R_OK): |
| os.makedirs(os.path.dirname(FILENAME), exist_ok=True) |
| with open(FILENAME, 'w') as f: |
| f.write('roboRIO-971.local') |
| with open(FILENAME, 'r') as f: |
| return f.readline().strip() |
| |
| def get_temp_dir(): |
| """Retrieves the temporary directory to use when downloading.""" |
| return '/home/admin/tmp/aos_downloader' |
| |
| def get_target_dir(): |
| """Retrieves the tempory deploy directory for downloading code.""" |
| return '/home/admin/robot_code' |
| |
| def user_output(message): |
| """Prints message to the user.""" |
| print('build.py: ' + message, file=sys.stderr) |
| |
| # A lock to avoid making a mess intermingling test-related messages. |
| test_output_lock = threading.RLock() |
| def test_output(message): |
| """Prints message to the user. Intended for messages related to tests.""" |
| with test_output_lock: |
| print('tests: ' + message, file=sys.stdout) |
| |
| def call_download_externals(argument): |
| """Calls download_externals.sh for a given set of externals. |
| |
| Args: |
| argument: The argument to pass to the shell script to tell it what to |
| download. |
| """ |
| subprocess.check_call( |
| (os.path.join(aos_path(), 'build', 'download_externals.sh'), |
| argument), |
| stdin=open(os.devnull, 'r')) |
| |
| class Processor(object): |
| """Represents a processor architecture we can build for.""" |
| |
| class UnknownPlatform(Exception): |
| def __init__(self, message): |
| super(Processor.UnknownPlatform, self).__init__() |
| self.message = message |
| |
| class Platform(object): |
| """Represents a single way to build the code.""" |
| |
| def outdir(self): |
| """Returns: |
| The path of the directory build outputs get put in to. |
| """ |
| return os.path.join( |
| aos_path(), '..', 'output', self.outname()) |
| def build_ninja(self): |
| """Returns: |
| The path of the build.ninja file. |
| """ |
| return os.path.join(self.outdir(), 'build.ninja') |
| |
| def do_deploy(self, dry_run, command): |
| """Helper for subclasses to implement deploy. |
| |
| Args: |
| dry_run: If True, prints the command instead of actually running it. |
| command: A tuple of command-line arguments. |
| """ |
| real_command = (('echo',) + command) if dry_run else command |
| subprocess.check_call(real_command, stdin=open(os.devnull, 'r')) |
| |
| def deploy(self, dry_run): |
| """Downloads the compiled code to the target computer.""" |
| raise NotImplementedError('deploy should be overriden') |
| def outname(self): |
| """Returns: |
| The name of the directory the code will be compiled to. |
| """ |
| raise NotImplementedError('outname should be overriden') |
| def os(self): |
| """Returns: |
| The name of the operating system this platform is for. |
| |
| This will be used as the value of the OS gyp variable. |
| """ |
| raise NotImplementedError('os should be overriden') |
| def gyp_platform(self): |
| """Returns: |
| The platform name the .gyp files know. |
| |
| This will be used as the value of the PLATFORM gyp variable. |
| """ |
| raise NotImplementedError('gyp_platform should be overriden') |
| def architecture(self): |
| """Returns: |
| The processor architecture for this platform. |
| |
| This will be used as the value of the ARCHITECTURE gyp variable. |
| """ |
| raise NotImplementedError('architecture should be overriden') |
| def compiler(self): |
| """Returns: |
| The compiler used for this platform. |
| |
| Everything before the first _ will be used as the value of the |
| COMPILER gyp variable and the whole thing will be used as the value |
| of the FULL_COMPILER gyp variable. |
| """ |
| raise NotImplementedError('compiler should be overriden') |
| def sanitizer(self): |
| """Returns: |
| The sanitizer used on this platform. |
| |
| This will be used as the value of the SANITIZER gyp variable. |
| |
| "none" if there isn't one. |
| """ |
| raise NotImplementedError('sanitizer should be overriden') |
| def debug(self): |
| """Returns: |
| Whether or not this platform compiles with debugging information. |
| |
| The DEBUG gyp variable will be set to "yes" or "no" based on this. |
| """ |
| raise NotImplementedError('debug should be overriden') |
| def build_env(self): |
| """Returns: |
| A map of environment variables to set while building this platform. |
| """ |
| raise NotImplementedError('build_env should be overriden') |
| def priority(self): |
| """Returns: |
| A relative priority for this platform relative to other ones. |
| |
| Higher priority platforms will get built, tested, etc first. Generally, |
| platforms which give higher-quality compiler errors etc should come first. |
| """ |
| return 0 |
| |
| def check_installed(self, platforms, is_deploy): |
| """Makes sure that everything necessary to build platforms are installed.""" |
| raise NotImplementedError('check_installed should be overriden') |
| def parse_platforms(self, platforms_string): |
| """Args: |
| string: A user-supplied string saying which platforms to select. |
| |
| Returns: |
| A tuple of Platform objects. |
| |
| Raises: |
| Processor.UnknownPlatform: Parsing string didn't work out. |
| """ |
| raise NotImplementedError('parse_platforms should be overriden') |
| def extra_gyp_flags(self): |
| """Returns: |
| A tuple of extra flags to pass to gyp (if any). |
| """ |
| return () |
| def modify_ninja_file(self, ninja_file): |
| """Modifies a freshly generated ninja file as necessary. |
| |
| Args: |
| ninja_file: Path to the file to modify. |
| """ |
| pass |
| def download_externals(self, platforms): |
| """Calls download_externals as appropriate to build platforms. |
| |
| Args: |
| platforms: A list of platforms to download external libraries for. |
| """ |
| raise NotImplementedError('download_externals should be overriden') |
| |
| def do_check_installed(self, other_packages): |
| """Helper for subclasses to implement check_installed. |
| |
| Args: |
| other_packages: A tuple of platform-specific packages to check for.""" |
| all_packages = other_packages |
| # Necessary to build stuff. |
| all_packages += ('ccache', 'make') |
| # Necessary to download stuff to build. |
| all_packages += ('wget', 'git', 'subversion', 'patch', 'unzip', 'bzip2') |
| # Necessary to build externals stuff. |
| all_packages += ('python', 'gcc', 'g++') |
| not_found = [] |
| try: |
| # TODO(brians): Check versions too. |
| result = subprocess.check_output( |
| ('dpkg-query', |
| r"--showformat='${binary:Package}\t${db:Status-Abbrev}\n'", |
| '--show') + all_packages, |
| stdin=open(os.devnull, 'r'), |
| stderr=subprocess.STDOUT) |
| for line in result.decode('utf-8').rstrip().splitlines(True): |
| match = re.match('^([^\t]+)\t[^i][^i]$', line) |
| if match: |
| not_found.append(match.group(1)) |
| except subprocess.CalledProcessError as e: |
| output = e.output.decode('utf-8').rstrip() |
| for line in output.splitlines(True): |
| match = re.match(r'dpkg-query: no packages found matching (.*)', |
| line) |
| if match: |
| not_found.append(match.group(1)) |
| if not_found: |
| user_output('Some packages not installed: %s.' % ', '.join(not_found)) |
| user_output('Try something like `sudo apt-get install %s`.' % |
| ' '.join(not_found)) |
| exit(1) |
| |
| class PrimeProcessor(Processor): |
| """A Processor subclass for building prime code.""" |
| |
| class Platform(Processor.Platform): |
| def __init__(self, architecture, compiler, debug, sanitizer): |
| super(PrimeProcessor.Platform, self).__init__() |
| |
| self.__architecture = architecture |
| self.__compiler = compiler |
| self.__debug = debug |
| self.__sanitizer = sanitizer |
| |
| def __repr__(self): |
| return 'PrimeProcessor.Platform(architecture=%s, compiler=%s, debug=%s' \ |
| ', sanitizer=%s)' \ |
| % (self.architecture(), self.compiler(), self.debug(), |
| self.sanitizer()) |
| def __str__(self): |
| return '%s-%s%s-%s' % (self.architecture(), self.compiler(), |
| '-debug' if self.debug() else '', self.sanitizer()) |
| |
| def os(self): |
| return 'linux' |
| def gyp_platform(self): |
| return '%s-%s-%s' % (self.os(), self.architecture(), self.compiler()) |
| def architecture(self): |
| return self.__architecture |
| def compiler(self): |
| return self.__compiler |
| def sanitizer(self): |
| return self.__sanitizer |
| def debug(self): |
| return self.__debug |
| |
| def outname(self): |
| return str(self) |
| |
| def priority(self): |
| r = 0 |
| if self.compiler() == 'clang': |
| r += 100 |
| if self.sanitizer() != 'none': |
| r -= 50 |
| elif self.debug(): |
| r -= 10 |
| if self.architecture() == 'amd64': |
| r += 5 |
| return r |
| |
| def deploy(self, dry_run): |
| """Downloads code to the prime in a way that avoids clashing too badly with |
| starter (like the naive download everything one at a time).""" |
| if not self.architecture().endswith('_frc'): |
| raise Exception("Don't know how to download code to a %s." % |
| self.architecture()) |
| SUM = 'md5sum' |
| TARGET_DIR = get_target_dir() |
| TEMP_DIR = get_temp_dir() |
| TARGET = 'admin@' + get_ip() |
| |
| from_dir = os.path.join(self.outdir(), 'outputs') |
| sums = subprocess.check_output((SUM,) + tuple(os.listdir(from_dir)), |
| stdin=open(os.devnull, 'r'), |
| cwd=from_dir) |
| to_download = subprocess.check_output( |
| ('ssh', TARGET, |
| """rm -rf {TMPDIR} && mkdir -p {TMPDIR} && \\ |
| mkdir -p {TO_DIR} && cd {TO_DIR} \\ |
| && echo '{SUMS}' | {SUM} -c \\ |
| |& grep -F FAILED | sed 's/^\\(.*\\): FAILED.*$/\\1/g'""". |
| format(TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR, SUMS=sums.decode('utf-8'), |
| SUM=SUM))) |
| if not to_download: |
| user_output("Nothing to download") |
| return |
| self.do_deploy( |
| dry_run, |
| ('scp', '-o', 'Compression yes') |
| + tuple([os.path.join(from_dir, f) for f in to_download.decode('utf-8').split('\n')[:-1]]) |
| + (('%s:%s' % (TARGET, TEMP_DIR)),)) |
| if not dry_run: |
| mv_cmd = ['mv {TMPDIR}/* {TO_DIR} '] |
| mv_cmd.append('&& chmod u+s {TO_DIR}/starter_exe ') |
| mv_cmd.append('&& echo \'Done moving new executables into place\' ') |
| mv_cmd.append('&& bash -c \'sync && sync && sync\'') |
| subprocess.check_call( |
| ('ssh', TARGET, |
| ''.join(mv_cmd).format(TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR))) |
| |
| def build_env(self): |
| OTHER_SYSROOT = '/usr/lib/llvm-3.5/' |
| SYMBOLIZER_PATH = OTHER_SYSROOT + 'bin/llvm-symbolizer' |
| r = {} |
| if self.sanitizer() == 'address': |
| r['ASAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH |
| r['ASAN_OPTIONS'] = \ |
| 'detect_leaks=1:check_initialization_order=1:strict_init_order=1' \ |
| ':detect_stack_use_after_return=1:detect_odr_violation=2' \ |
| ':allow_user_segv_handler=1' |
| elif self.sanitizer() == 'memory': |
| r['MSAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH |
| elif self.sanitizer() == 'thread': |
| r['TSAN_OPTIONS'] = 'external_symbolizer_path=' + SYMBOLIZER_PATH |
| # This is apparently the default for newer versions, which disagrees |
| # with documentation, so just turn it on explicitly. |
| r['TSAN_OPTIONS'] += ':detect_deadlocks=1' |
| # Print more useful stacks for mutex locking order problems. |
| r['TSAN_OPTIONS'] += ':second_deadlock_stack=1' |
| |
| r['CCACHE_COMPRESS'] = 'yes' |
| r['CCACHE_DIR'] = os.path.abspath(os.path.join(aos_path(), '..', 'output', |
| 'ccache_dir')) |
| r['CCACHE_HASHDIR'] = 'yes' |
| if self.compiler().startswith('clang'): |
| # clang doesn't like being run directly on the preprocessed files. |
| r['CCACHE_CPP2'] = 'yes' |
| # Without this, ccache slows down because of the generated header files. |
| # The race condition that this opens up isn't a problem because the build |
| # system finishes modifying header files before compiling anything that |
| # uses them. |
| r['CCACHE_SLOPPINESS'] = 'include_file_mtime' |
| |
| if self.architecture() == 'amd64': |
| r['PATH'] = os.path.join(aos_path(), 'build', 'bin-ld.gold') + \ |
| ':' + os.environ['PATH'] |
| |
| return r |
| |
| ARCHITECTURES = ('arm_frc', 'amd64') |
| COMPILERS = ('clang', 'gcc') |
| SANITIZERS = ('address', 'undefined', 'integer', 'memory', 'thread', 'none') |
| SANITIZER_TEST_WARNINGS = { |
| 'memory': (True, |
| """We don't have all of the libraries instrumented which leads to lots of false |
| errors with msan (especially stdlibc++). |
| TODO(brians): Figure out a way to deal with it."""), |
| } |
| PIE_SANITIZERS = ('memory', 'thread') |
| |
| def __init__(self, is_test, is_deploy): |
| super(PrimeProcessor, self).__init__() |
| |
| platforms = [] |
| for architecture in PrimeProcessor.ARCHITECTURES: |
| for compiler in PrimeProcessor.COMPILERS: |
| for debug in [True, False]: |
| if architecture == 'amd64' and compiler == 'gcc': |
| # We don't have a compiler to use here. |
| continue |
| platforms.append( |
| self.Platform(architecture, compiler, debug, 'none')) |
| for sanitizer in PrimeProcessor.SANITIZERS: |
| for compiler in ('clang',): |
| if compiler == 'gcc_4.8' and (sanitizer == 'undefined' or |
| sanitizer == 'integer' or |
| sanitizer == 'memory'): |
| # GCC 4.8 doesn't support these sanitizers. |
| continue |
| if sanitizer == 'none': |
| # We already added sanitizer == 'none' above. |
| continue |
| platforms.append( |
| self.Platform('amd64', compiler, True, sanitizer)) |
| self.__platforms = frozenset(platforms) |
| |
| if is_test: |
| default_platforms = self.select_platforms(architecture='amd64', |
| debug=True) |
| for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items(): |
| if warning[0]: |
| default_platforms -= self.select_platforms(sanitizer=sanitizer) |
| elif is_deploy: |
| default_platforms = self.select_platforms(architecture='arm_frc', |
| compiler='gcc', |
| debug=False) |
| else: |
| default_platforms = self.select_platforms(debug=False) |
| self.__default_platforms = frozenset(default_platforms) |
| |
| def platforms(self): |
| return self.__platforms |
| def default_platforms(self): |
| return self.__default_platforms |
| |
| def download_externals(self, platforms): |
| to_download = set() |
| for architecture in PrimeProcessor.ARCHITECTURES: |
| pie_sanitizers = set() |
| for sanitizer in PrimeProcessor.PIE_SANITIZERS: |
| pie_sanitizers.update(self.select_platforms(architecture=architecture, |
| sanitizer=sanitizer)) |
| if platforms & pie_sanitizers: |
| to_download.add(architecture + '-fPIE') |
| |
| if platforms & (self.select_platforms(architecture=architecture) - |
| pie_sanitizers): |
| to_download.add(architecture) |
| |
| for download_target in to_download: |
| call_download_externals(download_target) |
| |
| def parse_platforms(self, platform_string): |
| if platform_string is None: |
| return self.default_platforms() |
| r = self.default_platforms() |
| for part in platform_string.split(','): |
| if part == 'all': |
| r = self.platforms() |
| elif part[0] == '+': |
| r = r | self.select_platforms_string(part[1:]) |
| elif part[0] == '-': |
| r = r - self.select_platforms_string(part[1:]) |
| elif part[0] == '=': |
| r = self.select_platforms_string(part[1:]) |
| else: |
| selected = self.select_platforms_string(part) |
| r = r - (self.platforms() - selected) |
| if not r: |
| r = selected |
| return r |
| |
| def select_platforms(self, architecture=None, compiler=None, debug=None, |
| sanitizer=None): |
| r = [] |
| for platform in self.platforms(): |
| if architecture is None or platform.architecture() == architecture: |
| if compiler is None or platform.compiler() == compiler: |
| if debug is None or platform.debug() == debug: |
| if sanitizer is None or platform.sanitizer() == sanitizer: |
| r.append(platform) |
| return set(r) |
| |
| def select_platforms_string(self, platforms_string): |
| architecture, compiler, debug, sanitizer = None, None, None, None |
| for part in platforms_string.split('-'): |
| if part in PrimeProcessor.ARCHITECTURES: |
| architecture = part |
| elif part in PrimeProcessor.COMPILERS: |
| compiler = part |
| elif part in ['debug', 'dbg']: |
| debug = True |
| elif part in ['release', 'nodebug', 'ndb']: |
| debug = False |
| elif part in PrimeProcessor.SANITIZERS: |
| sanitizer = part |
| elif part == 'all': |
| architecture = compiler = debug = sanitizer = None |
| else: |
| raise Processor.UnknownPlatform( |
| '"%s" not recognized as a platform string component.' % part) |
| return self.select_platforms( |
| architecture=architecture, |
| compiler=compiler, |
| debug=debug, |
| sanitizer=sanitizer) |
| |
| def check_installed(self, platforms, is_deploy): |
| packages = set(('lzip', 'm4', 'realpath')) |
| packages.add('ruby') |
| packages.add('clang-3.5') |
| packages.add('clang-format-3.5') |
| for platform in platforms: |
| if platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8': |
| packages.add('clang-3.5') |
| if platform.compiler() == 'gcc_4.8': |
| packages.add('libcloog-isl3:amd64') |
| if is_deploy: |
| packages.add('openssh-client') |
| elif platform.architecture == 'arm_frc': |
| packages.add('gcc-4.9-arm-frc-linux-gnueabi') |
| packages.add('g++-4.9-arm-frc-linux-gnueabi') |
| |
| self.do_check_installed(tuple(packages)) |
| |
| class Bot3PrimeProcessor(PrimeProcessor): |
| """A very simple subclass of PrimeProcessor whose main function is to allow |
| the building of third robot targets in separate directories from those of |
| the main robot.""" |
| class Platform(PrimeProcessor.Platform): |
| def __str__(self): |
| return 'bot3-%s' % (super(Bot3PrimeProcessor.Platform, self).__str__()) |
| |
| |
| def strsignal(num): |
| # It ends up with SIGIOT instead otherwise, which is weird. |
| if num == signal.SIGABRT: |
| return 'SIGABRT' |
| # SIGCLD is a weird way to spell it. |
| if num == signal.SIGCHLD: |
| return 'SIGCHLD' |
| |
| SIGNALS_TO_NAMES = dict((getattr(signal, n), n) |
| for n in dir(signal) if n.startswith('SIG') |
| and '_' not in n) |
| return SIGNALS_TO_NAMES.get(num, 'Unknown signal %d' % num) |
| |
| def main(): |
| sys.argv.pop(0) |
| exec_name = sys.argv.pop(0) |
| def print_help(exit_status=None, message=None): |
| if message: |
| print(message) |
| sys.stdout.write( |
| """Usage: {name} [-j n] [action] [-n] [platform] [target|extra_flag]... |
| Arguments: |
| -j, --jobs Explicitly specify how many jobs to run at a time. |
| Defaults to the number of processors + 2. |
| -n, --dry-run Don't actually do whatever. |
| Currently only meaningful for deploy. |
| action What to do. Defaults to build. |
| build: Build the code. |
| clean: Remove all the built output. |
| tests: Build and then run tests. |
| deploy: Build and then download. |
| environment: Dump the environment for building. |
| platform What variants of the code to build. |
| Defaults to something reasonable. |
| See below for details. |
| target... Which targets to build/test/etc. |
| Defaults to everything. |
| extra_flag... Extra flags associated with the targets. |
| --gtest_*: Arguments to pass on to tests. |
| --print_logs, --log_file=*: More test arguments. |
| |
| Specifying targets: |
| Targets are combinations of architecture, compiler, and debug flags. Which |
| ones actually get run is built up as a set. It defaults to something |
| reasonable for the action (specified below). |
| The platform specification (the argument given to this script) is a comma- |
| separated sequence of hyphen-separated platforms, each with an optional |
| prefix. |
| Each selector (the things separated by commas) selects all of the platforms |
| which match all of its components. Its effect on the set of current platforms |
| depends on the prefix character. |
| Here are the prefix characters: |
| + Adds the selected platforms. |
| - Removes the selected platforms. |
| = Sets the current set to the selected platforms. |
| [none] Removes all non-selected platforms. |
| If this makes the current set empty, acts like =. |
| There is also the special psuedo-platform "all" which selects all platforms. |
| All of the available platforms: |
| {all_platforms} |
| Default platforms for deploying: |
| {deploy_platforms} |
| Default platforms for testing: |
| {test_platforms} |
| Default platforms for everything else: |
| {default_platforms} |
| |
| Examples of specifying targets: |
| build everything: "all" |
| only build things with clang: "clang" |
| build everything that uses GCC 4.8 (not just the defaults): "=gcc_4.8" |
| build all of the arm targets that use clang: "clang-arm" or "arm-clang" |
| """.format( |
| name=exec_name, |
| all_platforms=str_platforms(PrimeProcessor(False, False).platforms()), |
| deploy_platforms=str_platforms(PrimeProcessor(False, True).default_platforms()), |
| test_platforms=str_platforms(PrimeProcessor(True, False).default_platforms()), |
| default_platforms=str_platforms(PrimeProcessor(False, False).default_platforms()), |
| )) |
| if exit_status is not None: |
| sys.exit(exit_status) |
| |
| def sort_platforms(platforms): |
| return sorted( |
| platforms, key=lambda platform: (-platform.priority(), str(platform))) |
| |
| def str_platforms(platforms): |
| r = [] |
| for platform in sort_platforms(platforms): |
| r.append(str(platform)) |
| if len(r) > 1: |
| r[-1] = 'and ' + r[-1] |
| return ', '.join(r) |
| |
| class Arguments(object): |
| def __init__(self): |
| self.jobs = os.sysconf('SC_NPROCESSORS_ONLN') + 2 |
| self.action_name = 'build' |
| self.dry_run = False |
| self.targets = [] |
| self.platform = None |
| self.extra_flags = [] |
| |
| args = Arguments() |
| |
| if len(sys.argv) < 2: |
| print_help(1, 'Not enough arguments') |
| args.processor = sys.argv.pop(0) |
| args.main_gyp = sys.argv.pop(0) |
| VALID_ACTIONS = ['build', 'clean', 'deploy', 'tests', 'environment'] |
| while sys.argv: |
| arg = sys.argv.pop(0) |
| if arg == '-j' or arg == '--jobs': |
| args.jobs = int(sys.argv.pop(0)) |
| continue |
| if arg in VALID_ACTIONS: |
| args.action_name = arg |
| continue |
| if arg == '-n' or arg == '--dry-run': |
| if args.action_name != 'deploy': |
| print_help(1, '--dry-run is only valid for deploy') |
| args.dry_run = True |
| continue |
| if arg == '-h' or arg == '--help': |
| print_help(0) |
| if (re.match('^--gtest_.*$', arg) or arg == '--print-logs' or |
| re.match('^--log_file=.*$', arg)): |
| if args.action_name == 'tests': |
| args.extra_flags.append(arg) |
| continue |
| else: |
| print_help(1, '%s is only valid for tests' % arg) |
| if args.platform: |
| args.targets.append(arg) |
| else: |
| args.platform = arg |
| |
| if args.processor == 'prime': |
| processor = PrimeProcessor(args.action_name == 'tests', |
| args.action_name == 'deploy') |
| elif args.processor == 'bot3_prime': |
| processor = Bot3PrimeProcessor(args.action_name == 'tests', |
| args.action_name == 'deploy') |
| else: |
| print_help(1, message='Unknown processor "%s".' % args.processor) |
| |
| unknown_platform_error = None |
| try: |
| platforms = processor.parse_platforms(args.platform) |
| except Processor.UnknownPlatform as e: |
| unknown_platform_error = e.message |
| args.targets.insert(0, args.platform) |
| platforms = processor.parse_platforms(None) |
| if not platforms: |
| print_help(1, 'No platforms selected') |
| |
| processor.check_installed(platforms, args.action_name == 'deploy') |
| processor.download_externals(platforms) |
| |
| class ToolsConfig(object): |
| def __init__(self): |
| self.variables = {'AOS': aos_path()} |
| with open(os.path.join(aos_path(), 'build', 'tools_config'), 'r') as f: |
| for line in f: |
| if line[0] == '#': |
| pass |
| elif line.isspace(): |
| pass |
| else: |
| new_name, new_value = line.rstrip().split('=') |
| for name, value in self.variables.items(): |
| new_value = new_value.replace('${%s}' % name, value) |
| self.variables[new_name] = new_value |
| def __getitem__(self, key): |
| return self.variables[key] |
| |
| tools_config = ToolsConfig() |
| |
| def handle_clean_error(function, path, excinfo): |
| _, _ = function, path |
| if issubclass(OSError, excinfo[0]): |
| if excinfo[1].errno == errno.ENOENT: |
| # Who cares if the file we're deleting isn't there? |
| return |
| raise excinfo[1] |
| |
| def need_to_run_gyp(platform): |
| """Determines if we need to run gyp again or not. |
| |
| The generated build files are supposed to re-run gyp again themselves, but |
| that doesn't work (or at least it used to not) and we sometimes want to |
| modify the results anyways. |
| |
| Args: |
| platform: The platform to check for. |
| """ |
| if not os.path.exists(platform.build_ninja()): |
| return True |
| if os.path.getmtime(__file__) > os.path.getmtime(platform.build_ninja()): |
| return True |
| dirs = os.listdir(os.path.join(aos_path(), '..')) |
| # Looking through these folders takes a long time and isn't useful. |
| if dirs.count('output'): |
| dirs.remove('output') |
| if dirs.count('.git'): |
| dirs.remove('.git') |
| return not not subprocess.check_output( |
| ('find',) + tuple(os.path.join(aos_path(), '..', d) for d in dirs) |
| + ('-newer', platform.build_ninja(), |
| '(', '-name', '*.gyp', '-or', '-name', '*.gypi', ')'), |
| stdin=open(os.devnull, 'r')) |
| |
| def env(platform): |
| """Makes sure we pass through important environmental variables. |
| |
| Returns: |
| An environment suitable for passing to subprocess.Popen and friends. |
| """ |
| build_env = dict(platform.build_env()) |
| if not 'TERM' in build_env: |
| build_env['TERM'] = os.environ['TERM'] |
| if not 'PATH' in build_env: |
| build_env['PATH'] = os.environ['PATH'] |
| return build_env |
| |
| sorted_platforms = sort_platforms(platforms) |
| user_output('Building %s...' % str_platforms(sorted_platforms)) |
| |
| if args.action_name == 'tests': |
| for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items(): |
| warned_about = platforms & processor.select_platforms(sanitizer=sanitizer) |
| if warned_about: |
| user_output(warning[1]) |
| if warning[0]: |
| # TODO(brians): Add a --force flag or something to override this? |
| user_output('Refusing to run tests for sanitizer %s.' % sanitizer) |
| exit(1) |
| |
| num = 1 |
| for platform in sorted_platforms: |
| user_output('Building %s (%d/%d)...' % (platform, num, len(platforms))) |
| if args.action_name == 'clean': |
| shutil.rmtree(platform.outdir(), onerror=handle_clean_error) |
| elif args.action_name == 'environment': |
| user_output('Environment for building <<END') |
| for name, value in env(platform).items(): |
| print('%s=%s' % (name, value)) |
| print('END') |
| else: |
| if need_to_run_gyp(platform): |
| user_output('Running gyp...') |
| gyp = subprocess.Popen( |
| (tools_config['GYP'], |
| '--check', |
| '--depth=%s' % os.path.join(aos_path(), '..'), |
| '--no-circular-check', |
| '-f', 'ninja', |
| '-I%s' % os.path.join(aos_path(), 'build', 'aos.gypi'), |
| '-I/dev/stdin', '-Goutput_dir=output', |
| '-DOS=%s' % platform.os(), |
| '-DPLATFORM=%s' % platform.gyp_platform(), |
| '-DARCHITECTURE=%s' % platform.architecture(), |
| '-DCOMPILER=%s' % platform.compiler().split('_')[0], |
| '-DFULL_COMPILER=%s' % platform.compiler(), |
| '-DDEBUG=%s' % ('yes' if platform.debug() else 'no'), |
| '-DSANITIZER=%s' % platform.sanitizer(), |
| '-DEXTERNALS_EXTRA=%s' % |
| ('-fPIE' if platform.sanitizer() in PrimeProcessor.PIE_SANITIZERS |
| else '')) + |
| processor.extra_gyp_flags() + (args.main_gyp,), |
| stdin=subprocess.PIPE) |
| gyp.communicate((""" |
| { |
| 'target_defaults': { |
| 'configurations': { |
| '%s': {} |
| } |
| } |
| }""" % platform.outname()).encode()) |
| if gyp.returncode: |
| user_output("Running gyp failed!") |
| exit(1) |
| processor.modify_ninja_file(platform.build_ninja()) |
| user_output('Done running gyp') |
| else: |
| user_output("Not running gyp") |
| |
| try: |
| call = (tools_config['NINJA'], |
| '-C', platform.outdir()) + tuple(args.targets) |
| if args.jobs: |
| call += ('-j', str(args.jobs)) |
| subprocess.check_call(call, |
| stdin=open(os.devnull, 'r'), |
| env=env(platform)) |
| except subprocess.CalledProcessError as e: |
| if unknown_platform_error is not None: |
| user_output(unknown_platform_error) |
| raise e |
| |
| if args.action_name == 'deploy': |
| platform.deploy(args.dry_run) |
| elif args.action_name == 'tests': |
| dirname = os.path.join(platform.outdir(), 'tests') |
| done_queue = queue.Queue() |
| running = [] |
| test_start_semaphore = threading.Semaphore(args.jobs) |
| if args.targets: |
| to_run = [] |
| for target in args.targets: |
| if target.endswith('_test'): |
| to_run.append(target) |
| else: |
| to_run = os.listdir(dirname) |
| for f in to_run: |
| thread = TestThread(os.path.join(dirname, f), tuple(args.extra_flags), |
| env(platform), done_queue, |
| test_start_semaphore) |
| running.append(thread) |
| thread.start() |
| try: |
| while running: |
| done = done_queue.get() |
| running.remove(done) |
| with test_output_lock: |
| test_output('Output from test %s:' % done.name) |
| try: |
| while True: |
| line = done.output.get(False) |
| if not sys.stdout.isatty(): |
| # Remove color escape codes. |
| line = re.sub(r'\x1B\[[0-9;]*[a-zA-Z]', '', line) |
| sys.stdout.write(line) |
| except queue.Empty: |
| pass |
| if not done.returncode: |
| test_output('Test %s succeeded' % done.name) |
| else: |
| if done.returncode < 0: |
| sig = -done.returncode |
| test_output('Test %s was killed by signal %d (%s)' % \ |
| (done.name, sig, strsignal(sig))) |
| elif done.returncode != 1: |
| test_output('Test %s exited with %d' % \ |
| (done.name, done.returncode)) |
| else: |
| test_output('Test %s failed' % done.name) |
| user_output('Aborting because of test failure for %s.' % \ |
| platform) |
| exit(1) |
| finally: |
| if running: |
| test_output('Killing other tests...') |
| # Stop all of them before killing processes because otherwise stopping some of |
| # them tends to let other ones that are waiting to start go. |
| for thread in running: |
| thread.stop() |
| for thread in running: |
| test_output('\tKilling %s' % thread.name) |
| thread.kill_process() |
| thread.kill_process() |
| test_output('Waiting for other tests to die') |
| for thread in running: |
| thread.kill_process() |
| thread.join() |
| test_output('Done killing other tests') |
| |
| user_output('Done building %s (%d/%d)' % (platform, num, len(platforms))) |
| num += 1 |
| |
| if __name__ == '__main__': |
| main() |