blob: a36f69cd04da198172addfc6ef83544de8112edd [file] [log] [blame]
Brian Silvermana29ebf92014-04-23 13:08:49 -05001#!/usr/bin/python3
2
Brian Silvermana29ebf92014-04-23 13:08:49 -05003import sys
4import subprocess
5import re
6import os
7import os.path
8import string
9import shutil
10import errno
Brian Silvermanc3740c32014-05-04 12:42:47 -070011import queue
12import threading
Brian Silvermanf2bbe092014-05-13 16:55:03 -070013import pty
Brian Silverman6bca4722014-05-20 17:02:49 -070014import signal
Brian Silvermanc3740c32014-05-04 12:42:47 -070015
16class TestThread(threading.Thread):
Brian Silvermane6bada62014-05-04 16:16:54 -070017 """Runs 1 test and keeps track of its current state.
18
19 A TestThread is either waiting to start the test, actually running it, done,
20 running it, or stopped. The first 3 always happen in that order and can
21 change to stopped at any time.
22
23 It will finish (ie join() will return) once the process has exited, at which
24 point accessing process to see the status is OK.
25
26 Attributes:
27 executable: The file path of the executable to run.
Brian Silvermanb9e89602014-06-27 14:21:08 -050028 args: A tuple of arguments to give the executable.
Brian Silvermane6bada62014-05-04 16:16:54 -070029 env: The environment variables to set.
30 done_queue: A queue.Queue to place self on once done running the test.
31 start_semaphore: A threading.Semaphore to wait on before starting.
32 process_lock: A lock around process.
33 process: The currently executing test process or None. Synchronized by
34 process_lock.
35 stopped: True if we're stopped.
Brian Silverman730bb012014-06-08 13:05:20 -070036 output: A queue of lines of output from the test.
Brian Silvermane6bada62014-05-04 16:16:54 -070037 """
Brian Silverman730bb012014-06-08 13:05:20 -070038
39 class OutputCopier(threading.Thread):
40 """Copies the output of a test from its output pty into a queue.
41
42 This is necessary because otherwise everything locks up if the test writes
43 too much output and fills up the pty's buffer.
44 """
45
46 def __init__(self, name, fd, queue):
47 super(TestThread.OutputCopier, self).__init__(
48 name=(name + '.OutputCopier'))
49
50 self.fd = fd
51 self.queue = queue
52
53 def run(self):
54 with os.fdopen(self.fd) as to_read:
55 try:
56 for line in to_read:
57 self.queue.put(line)
58 except IOError as e:
59# An EIO from the master side of the pty means we hit the end.
60 if e.errno == errno.EIO:
61 return
62 else:
63 raise e
64
Brian Silvermanb9e89602014-06-27 14:21:08 -050065 def __init__(self, executable, args, env, done_queue, start_semaphore):
Brian Silvermanc3740c32014-05-04 12:42:47 -070066 super(TestThread, self).__init__(
Brian Silvermanb9e89602014-06-27 14:21:08 -050067 name=os.path.split(executable)[-1])
Brian Silvermanc3740c32014-05-04 12:42:47 -070068
69 self.executable = executable
Brian Silvermanb9e89602014-06-27 14:21:08 -050070 self.args = args
Brian Silvermanc3740c32014-05-04 12:42:47 -070071 self.env = env
72 self.done_queue = done_queue
73 self.start_semaphore = start_semaphore
74
Brian Silverman730bb012014-06-08 13:05:20 -070075 self.output = queue.Queue()
76
Brian Silvermanc3740c32014-05-04 12:42:47 -070077 self.process_lock = threading.Lock()
78 self.process = None
79 self.stopped = False
Brian Silverman452aaec2014-05-05 16:52:18 -070080 self.returncode = None
Brian Silverman730bb012014-06-08 13:05:20 -070081 self.output_copier = None
Brian Silvermanc3740c32014-05-04 12:42:47 -070082
83 def run(self):
84 with self.start_semaphore:
Brian Silverman452aaec2014-05-05 16:52:18 -070085 if self.stopped:
86 return
Brian Silvermanc3740c32014-05-04 12:42:47 -070087 test_output('Starting test %s...' % self.name)
Brian Silverman730bb012014-06-08 13:05:20 -070088 output_to_read, subprocess_output = pty.openpty()
89 self.output_copier = TestThread.OutputCopier(self.name, output_to_read,
90 self.output)
91 self.output_copier.start()
Brian Silvermanf2bbe092014-05-13 16:55:03 -070092 try:
93 with self.process_lock:
Brian Silvermanb9e89602014-06-27 14:21:08 -050094 self.process = subprocess.Popen((self.name,) + self.args,
95 executable=self.executable,
Brian Silvermanf2bbe092014-05-13 16:55:03 -070096 env=self.env,
97 stderr=subprocess.STDOUT,
98 stdout=subprocess_output,
99 stdin=open(os.devnull, 'r'))
100 finally:
101 os.close(subprocess_output)
Brian Silvermanc3740c32014-05-04 12:42:47 -0700102 self.process.wait()
103 with self.process_lock:
104 self.returncode = self.process.returncode
105 self.process = None
106 if not self.stopped:
Brian Silverman730bb012014-06-08 13:05:20 -0700107 self.output_copier.join()
Brian Silvermanc3740c32014-05-04 12:42:47 -0700108 self.done_queue.put(self)
109
110 def terminate_process(self):
Brian Silvermanbf0e1db2014-05-10 22:13:15 -0700111 """Asks any currently running process to stop."""
Brian Silvermanc3740c32014-05-04 12:42:47 -0700112 with self.process_lock:
Brian Silverman452aaec2014-05-05 16:52:18 -0700113 if not self.process:
114 return
Brian Silvermana5826582014-06-03 19:46:35 -0700115 try:
116 self.process.terminate()
117 except OSError as e:
118 if e.errno == errno.ESRCH:
119 # We don't really care if it's already gone.
120 pass
121 else:
122 raise e
Brian Silvermanc3740c32014-05-04 12:42:47 -0700123 def kill_process(self):
Brian Silvermanbf0e1db2014-05-10 22:13:15 -0700124 """Forcibly terminates any running process."""
Brian Silvermanc3740c32014-05-04 12:42:47 -0700125 with self.process_lock:
Brian Silverman452aaec2014-05-05 16:52:18 -0700126 if not self.process:
127 return
Brian Silverman6bca4722014-05-20 17:02:49 -0700128 try:
129 self.process.kill()
130 except OSError as e:
131 if e.errno == errno.ESRCH:
132 # We don't really care if it's already gone.
133 pass
134 else:
135 raise e
Brian Silvermanbf0e1db2014-05-10 22:13:15 -0700136 def stop(self):
137 """Changes self to the stopped state."""
138 with self.process_lock:
139 self.stopped = True
Brian Silvermana29ebf92014-04-23 13:08:49 -0500140
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500141def aos_path():
Brian Silvermane6bada62014-05-04 16:16:54 -0700142 """Returns:
143 A relative path to the aos directory.
144 """
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500145 return os.path.join(os.path.dirname(__file__), '..')
146
Austin Schuhd3680052014-10-25 17:52:18 -0700147def get_ip_base():
148 """Retrieves the IP address base."""
Brian Silvermane6bada62014-05-04 16:16:54 -0700149 FILENAME = os.path.normpath(os.path.join(aos_path(), '..',
Brian Silverman452aaec2014-05-05 16:52:18 -0700150 'output', 'ip_base.txt'))
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500151 if not os.access(FILENAME, os.R_OK):
152 os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
153 with open(FILENAME, 'w') as f:
154 f.write('10.9.71')
155 with open(FILENAME, 'r') as f:
Brian Silvermand043d472014-06-21 15:32:39 -0700156 base = f.readline().strip()
Austin Schuhd3680052014-10-25 17:52:18 -0700157 return base
158
159def get_ip(device):
160 """Retrieves the IP address for a given device."""
161 base = get_ip_base()
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500162 if device == 'prime':
163 return base + '.179'
164 elif device == 'robot':
165 return base + '.2'
Austin Schuh93426072014-10-21 22:22:06 -0700166 elif device == 'roboRIO':
167 return base + '.2'
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500168 else:
169 raise Exception('Unknown device %s to get an IP address for.' % device)
170
Austin Schuhd3680052014-10-25 17:52:18 -0700171def get_user(device):
172 """Retrieves the user for a given device."""
173 if device == 'prime':
174 return 'driver'
175 elif device == 'roboRIO':
176 return 'admin'
177 else:
178 raise Exception('Unknown device %s to get a user for.' % device)
179
180def get_temp_dir(device):
181 """Retrieves the temporary download directory for a given device."""
182 if device == 'prime':
183 return '/tmp/aos_downloader'
184 elif device == 'roboRIO':
185 return '/home/admin/tmp/aos_downloader'
186 else:
187 raise Exception('Unknown device %s to get a temp_dir for.' % device)
188
189def get_target_dir(device):
190 """Retrieves the tempory deploy directory for a given device."""
191 if device == 'prime':
192 return '/home/driver/robot_code/bin'
193 elif device == 'roboRIO':
194 return '/home/admin/robot_code'
195 else:
196 raise Exception('Unknown device %s to get a temp_dir for.' % device)
197
Brian Silvermana9b1e5c2014-04-30 18:08:04 -0700198def user_output(message):
Brian Silvermane6bada62014-05-04 16:16:54 -0700199 """Prints message to the user."""
Brian Silvermana9b1e5c2014-04-30 18:08:04 -0700200 print('build.py: ' + message, file=sys.stderr)
201
Brian Silverman452aaec2014-05-05 16:52:18 -0700202# A lock to avoid making a mess intermingling test-related messages.
Brian Silvermanc3740c32014-05-04 12:42:47 -0700203test_output_lock = threading.RLock()
204def test_output(message):
Brian Silvermane6bada62014-05-04 16:16:54 -0700205 """Prints message to the user. Intended for messages related to tests."""
Brian Silvermanc3740c32014-05-04 12:42:47 -0700206 with test_output_lock:
Brian Silvermane6bada62014-05-04 16:16:54 -0700207 print('tests: ' + message, file=sys.stdout)
208
209def call_download_externals(argument):
210 """Calls download_externals.sh for a given set of externals.
211
212 Args:
213 argument: The argument to pass to the shell script to tell it what to
214 download.
215 """
216 subprocess.check_call(
217 (os.path.join(aos_path(), 'build', 'download_externals.sh'),
218 argument),
219 stdin=open(os.devnull, 'r'))
Brian Silvermanc3740c32014-05-04 12:42:47 -0700220
Brian Silvermana29ebf92014-04-23 13:08:49 -0500221class Processor(object):
Brian Silvermane6bada62014-05-04 16:16:54 -0700222 """Represents a processor architecture we can build for."""
223
Brian Silvermana29ebf92014-04-23 13:08:49 -0500224 class UnknownPlatform(Exception):
225 def __init__(self, message):
Brian Silverman452aaec2014-05-05 16:52:18 -0700226 super(Processor.UnknownPlatform, self).__init__()
Brian Silvermana29ebf92014-04-23 13:08:49 -0500227 self.message = message
228
Brian Silvermanb3d50542014-04-23 14:28:55 -0500229 class Platform(object):
Brian Silvermane6bada62014-05-04 16:16:54 -0700230 """Represents a single way to build the code."""
231
Brian Silvermanb3d50542014-04-23 14:28:55 -0500232 def outdir(self):
Brian Silvermane6bada62014-05-04 16:16:54 -0700233 """Returns:
234 The path of the directory build outputs get put in to.
235 """
Brian Silvermanb3d50542014-04-23 14:28:55 -0500236 return os.path.join(
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500237 aos_path(), '..', 'output', self.outname())
Brian Silvermanb3d50542014-04-23 14:28:55 -0500238 def build_ninja(self):
Brian Silvermane6bada62014-05-04 16:16:54 -0700239 """Returns:
240 The path of the build.ninja file.
241 """
Brian Silvermanb3d50542014-04-23 14:28:55 -0500242 return os.path.join(self.outdir(), 'build.ninja')
243
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500244 def do_deploy(self, dry_run, command):
Brian Silvermane6bada62014-05-04 16:16:54 -0700245 """Helper for subclasses to implement deploy.
246
247 Args:
248 dry_run: If True, prints the command instead of actually running it.
249 command: A tuple of command-line arguments.
250 """
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500251 real_command = (('echo',) + command) if dry_run else command
252 subprocess.check_call(real_command, stdin=open(os.devnull, 'r'))
Brian Silvermana29ebf92014-04-23 13:08:49 -0500253
Brian Silvermane6bada62014-05-04 16:16:54 -0700254 def deploy(self, dry_run):
255 """Downloads the compiled code to the target computer."""
256 raise NotImplementedError('deploy should be overriden')
257 def outname(self):
258 """Returns:
259 The name of the directory the code will be compiled to.
260 """
261 raise NotImplementedError('outname should be overriden')
262 def os(self):
263 """Returns:
264 The name of the operating system this platform is for.
265
266 This will be used as the value of the OS gyp variable.
267 """
268 raise NotImplementedError('os should be overriden')
269 def gyp_platform(self):
270 """Returns:
271 The platform name the .gyp files know.
272
273 This will be used as the value of the PLATFORM gyp variable.
274 """
275 raise NotImplementedError('gyp_platform should be overriden')
276 def architecture(self):
277 """Returns:
278 The processor architecture for this platform.
279
280 This will be used as the value of the ARCHITECTURE gyp variable.
281 """
282 raise NotImplementedError('architecture should be overriden')
283 def compiler(self):
284 """Returns:
285 The compiler used for this platform.
286
287 Everything before the first _ will be used as the value of the
288 COMPILER gyp variable and the whole thing will be used as the value
289 of the FULL_COMPILER gyp variable.
290 """
291 raise NotImplementedError('compiler should be overriden')
Brian Silverman452aaec2014-05-05 16:52:18 -0700292 def sanitizer(self):
293 """Returns:
294 The sanitizer used on this platform.
295
296 This will be used as the value of the SANITIZER gyp variable.
297
298 "none" if there isn't one.
299 """
300 raise NotImplementedError('sanitizer should be overriden')
Brian Silvermane6bada62014-05-04 16:16:54 -0700301 def debug(self):
302 """Returns:
303 Whether or not this platform compiles with debugging information.
304
305 The DEBUG gyp variable will be set to "yes" or "no" based on this.
306 """
307 raise NotImplementedError('debug should be overriden')
308 def build_env(self):
309 """Returns:
310 A map of environment variables to set while building this platform.
311 """
312 raise NotImplementedError('build_env should be overriden')
Brian Silvermanbd380fd2014-05-13 16:55:24 -0700313 def priority(self):
314 """Returns:
315 A relative priority for this platform relative to other ones.
316
317 Higher priority platforms will get built, tested, etc first. Generally,
318 platforms which give higher-quality compiler errors etc should come first.
319 """
320 return 0
Brian Silvermane6bada62014-05-04 16:16:54 -0700321
Brian Silverman9b7a6842014-05-05 16:19:11 -0700322 def check_installed(self, platforms, is_deploy):
323 """Makes sure that everything necessary to build platforms are installed."""
Brian Silvermane6bada62014-05-04 16:16:54 -0700324 raise NotImplementedError('check_installed should be overriden')
Brian Silverman452aaec2014-05-05 16:52:18 -0700325 def parse_platforms(self, platforms_string):
Brian Silvermane6bada62014-05-04 16:16:54 -0700326 """Args:
327 string: A user-supplied string saying which platforms to select.
328
329 Returns:
330 A tuple of Platform objects.
331
332 Raises:
333 Processor.UnknownPlatform: Parsing string didn't work out.
334 """
335 raise NotImplementedError('parse_platforms should be overriden')
336 def extra_gyp_flags(self):
337 """Returns:
338 A tuple of extra flags to pass to gyp (if any).
339 """
340 return ()
341 def modify_ninja_file(self, ninja_file):
342 """Modifies a freshly generated ninja file as necessary.
343
344 Args:
345 ninja_file: Path to the file to modify.
346 """
347 pass
348 def download_externals(self, platforms):
349 """Calls download_externals as appropriate to build platforms.
350
351 Args:
352 platforms: A list of platforms to download external libraries for.
353 """
354 raise NotImplementedError('download_externals should be overriden')
355
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700356 def do_check_installed(self, other_packages):
Brian Silvermane6bada62014-05-04 16:16:54 -0700357 """Helper for subclasses to implement check_installed.
358
359 Args:
360 other_packages: A tuple of platform-specific packages to check for."""
Brian Silverman9b7a6842014-05-05 16:19:11 -0700361 all_packages = other_packages
362 # Necessary to build stuff.
363 all_packages += ('ccache', 'make')
364 # Necessary to download stuff to build.
365 all_packages += ('wget', 'git', 'subversion', 'patch', 'unzip', 'bzip2')
366 # Necessary to build externals stuff.
367 all_packages += ('python', 'gcc', 'g++')
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700368 try:
Brian Silverman452aaec2014-05-05 16:52:18 -0700369 # TODO(brians): Check versions too.
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700370 result = subprocess.check_output(
371 ('dpkg-query', '--show') + all_packages,
372 stdin=open(os.devnull, 'r'),
373 stderr=subprocess.STDOUT)
374 except subprocess.CalledProcessError as e:
Brian Silverman9b7a6842014-05-05 16:19:11 -0700375 output = e.output.decode('utf-8').rstrip()
376 not_found = []
377 for line in output.splitlines(True):
378 match = re.match(r'dpkg-query: no packages found matching (.*)',
379 line)
380 if match:
381 not_found.append(match.group(1))
382 user_output('Some packages not installed: %s.' % ', '.join(not_found))
383 user_output('Try something like `sudo apt-get install %s`.' %
Brian Silverman452aaec2014-05-05 16:52:18 -0700384 ' '.join(not_found))
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700385 exit(1)
386
Brian Silvermana29ebf92014-04-23 13:08:49 -0500387class CRIOProcessor(Processor):
Brian Silvermane6bada62014-05-04 16:16:54 -0700388 """A Processor subclass for building cRIO code."""
389
Brian Silvermanb3d50542014-04-23 14:28:55 -0500390 class Platform(Processor.Platform):
Brian Silvermana4aff562014-05-02 17:43:50 -0700391 def __init__(self, debug, wind_base):
Brian Silvermanb3d50542014-04-23 14:28:55 -0500392 super(CRIOProcessor.Platform, self).__init__()
393
Brian Silvermane6bada62014-05-04 16:16:54 -0700394 self.__debug = debug
395 self.__wind_base = wind_base
Brian Silvermanb3d50542014-04-23 14:28:55 -0500396
397 def __repr__(self):
Brian Silverman9b7a6842014-05-05 16:19:11 -0700398 return 'CRIOProcessor.Platform(debug=%s)' % self.debug()
Brian Silvermanb3d50542014-04-23 14:28:55 -0500399 def __str__(self):
Brian Silverman9b7a6842014-05-05 16:19:11 -0700400 return 'crio%s' % ('-debug' if self.debug() else '')
Brian Silvermanb3d50542014-04-23 14:28:55 -0500401
402 def outname(self):
Brian Silverman9b7a6842014-05-05 16:19:11 -0700403 return 'crio-debug' if self.debug() else 'crio'
Brian Silvermanb3d50542014-04-23 14:28:55 -0500404 def os(self):
405 return 'vxworks'
406 def gyp_platform(self):
407 return 'crio'
408 def architecture(self):
409 return 'ppc'
410 def compiler(self):
411 return 'gcc'
Brian Silverman9b7a6842014-05-05 16:19:11 -0700412 def sanitizer(self):
413 return 'none'
Brian Silvermane6bada62014-05-04 16:16:54 -0700414 def debug(self):
415 return self.__debug
416 def wind_base(self):
417 return self.__wind_base
Brian Silvermanb3d50542014-04-23 14:28:55 -0500418
Brian Silvermane48c09a2014-04-30 18:04:58 -0700419 # TODO(brians): test this
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500420 def deploy(self, dry_run):
421 self.do_deploy(dry_run,
422 ('ncftpput', get_ip('robot'), '/',
Brian Silverman452aaec2014-05-05 16:52:18 -0700423 os.path.join(self.outdir(), 'lib',
424 'FRC_UserProgram.out')))
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500425
Brian Silvermana4aff562014-05-02 17:43:50 -0700426 def build_env(self):
Brian Silvermane6bada62014-05-04 16:16:54 -0700427 return {'WIND_BASE': self.wind_base()}
Brian Silvermana4aff562014-05-02 17:43:50 -0700428
Brian Silvermana29ebf92014-04-23 13:08:49 -0500429 def __init__(self):
430 super(CRIOProcessor, self).__init__()
431
432 if 'WIND_BASE' in os.environ:
Brian Silvermane6bada62014-05-04 16:16:54 -0700433 self.__wind_base = os.environ['WIND_BASE']
Brian Silvermana29ebf92014-04-23 13:08:49 -0500434 else:
Brian Silvermane6bada62014-05-04 16:16:54 -0700435 self.__wind_base = '/usr/local/powerpc-wrs-vxworks/wind_base'
Brian Silvermana29ebf92014-04-23 13:08:49 -0500436
Brian Silverman452aaec2014-05-05 16:52:18 -0700437 def parse_platforms(self, platforms_string):
438 if platforms_string is None or platforms_string == 'crio':
Brian Silvermane6bada62014-05-04 16:16:54 -0700439 return (CRIOProcessor.Platform(False, self.wind_base()),)
Brian Silvermana4aff562014-05-02 17:43:50 -0700440 elif string == 'crio-debug' or string == 'debug':
Brian Silvermane6bada62014-05-04 16:16:54 -0700441 return (CRIOProcessor.Platform(True, self.wind_base()),)
Brian Silvermanb3d50542014-04-23 14:28:55 -0500442 else:
Brian Silverman452aaec2014-05-05 16:52:18 -0700443 raise Processor.UnknownPlatform(
Brian Silvermandf5348a2014-06-12 23:25:08 -0700444 '"%s" not recognized as a cRIO platform.' % platforms_string)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500445
Brian Silvermane6bada62014-05-04 16:16:54 -0700446 def wind_base(self):
447 return self.__wind_base
Brian Silvermanb3d50542014-04-23 14:28:55 -0500448
Brian Silvermane6bada62014-05-04 16:16:54 -0700449 def extra_gyp_flags(self):
450 return ('-DWIND_BASE=%s' % self.wind_base(),)
451
452 def modify_ninja_file(self, ninja_file):
453 subprocess.check_call(
454 ('sed', '-i',
455 's/nm -gD/nm/g', ninja_file),
456 stdin=open(os.devnull, 'r'))
457
458 def download_externals(self, _):
459 call_download_externals('crio')
Brian Silvermana29ebf92014-04-23 13:08:49 -0500460
Brian Silverman9b7a6842014-05-05 16:19:11 -0700461 def check_installed(self, platforms, is_deploy):
462 packages = ('powerpc-wrs-vxworks', 'tcl')
463 if is_deploy:
464 packages += ('ncftp',)
465 self.do_check_installed(packages)
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700466
Brian Silvermana29ebf92014-04-23 13:08:49 -0500467class PrimeProcessor(Processor):
Brian Silvermane6bada62014-05-04 16:16:54 -0700468 """A Processor subclass for building prime code."""
469
Brian Silvermanb3d50542014-04-23 14:28:55 -0500470 class Platform(Processor.Platform):
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700471 def __init__(self, architecture, compiler, debug, sanitizer):
Brian Silvermanb3d50542014-04-23 14:28:55 -0500472 super(PrimeProcessor.Platform, self).__init__()
473
Brian Silvermane6bada62014-05-04 16:16:54 -0700474 self.__architecture = architecture
475 self.__compiler = compiler
476 self.__debug = debug
477 self.__sanitizer = sanitizer
Brian Silvermana29ebf92014-04-23 13:08:49 -0500478
479 def __repr__(self):
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700480 return 'PrimeProcessor.Platform(architecture=%s, compiler=%s, debug=%s' \
481 ', sanitizer=%s)' \
Brian Silvermane6bada62014-05-04 16:16:54 -0700482 % (self.architecture(), self.compiler(), self.debug(),
483 self.sanitizer())
Brian Silvermana29ebf92014-04-23 13:08:49 -0500484 def __str__(self):
Brian Silvermane6bada62014-05-04 16:16:54 -0700485 return '%s-%s%s-%s' % (self.architecture(), self.compiler(),
Brian Silverman452aaec2014-05-05 16:52:18 -0700486 '-debug' if self.debug() else '', self.sanitizer())
Brian Silvermana29ebf92014-04-23 13:08:49 -0500487
488 def os(self):
489 return 'linux'
490 def gyp_platform(self):
Brian Silvermane6bada62014-05-04 16:16:54 -0700491 return '%s-%s-%s' % (self.os(), self.architecture(), self.compiler())
492 def architecture(self):
493 return self.__architecture
494 def compiler(self):
495 return self.__compiler
496 def sanitizer(self):
497 return self.__sanitizer
498 def debug(self):
499 return self.__debug
Brian Silvermana29ebf92014-04-23 13:08:49 -0500500
Brian Silvermana29ebf92014-04-23 13:08:49 -0500501 def outname(self):
502 return str(self)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500503
Brian Silvermanbd380fd2014-05-13 16:55:24 -0700504 def priority(self):
505 r = 0
506 if self.compiler() == 'gcc':
507 r -= 100
508 elif self.compiler() == 'clang':
509 r += 100
510 if self.sanitizer() != 'none':
511 r -= 50
512 elif self.debug():
513 r -= 10
514 if self.architecture() == 'amd64':
515 r += 5
516 return r
517
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500518 def deploy(self, dry_run):
Brian Silvermane6bada62014-05-04 16:16:54 -0700519 # Downloads code to the prime in a way that avoids clashing too badly with
520 # starter (like the naive download everything one at a time).
Austin Schuhd3680052014-10-25 17:52:18 -0700521 if self.compiler() == 'gcc_frc':
522 device = 'roboRIO'
523 else:
524 device = 'prime'
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500525 SUM = 'md5sum'
Austin Schuhd3680052014-10-25 17:52:18 -0700526 TARGET_DIR = get_target_dir(device)
527 TEMP_DIR = get_temp_dir(device)
528 TARGET = get_user(device) + '@' + get_ip(device)
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500529
530 from_dir = os.path.join(self.outdir(), 'outputs')
531 sums = subprocess.check_output((SUM,) + tuple(os.listdir(from_dir)),
532 stdin=open(os.devnull, 'r'),
533 cwd=from_dir)
534 to_download = subprocess.check_output(
535 ('ssh', TARGET,
Austin Schuh93426072014-10-21 22:22:06 -0700536 """rm -rf {TMPDIR} && mkdir -p {TMPDIR} && cd {TO_DIR} \\
537 && echo '{SUMS}' | {SUM} -c \\
Brian Silvermanff485782014-06-18 19:59:09 -0700538 |& grep -F FAILED | sed 's/^\\(.*\\): FAILED.*$/\\1/g'""".
539 format(TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR, SUMS=sums.decode('utf-8'),
540 SUM=SUM)))
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500541 if not to_download:
Brian Silvermana9b1e5c2014-04-30 18:08:04 -0700542 user_output("Nothing to download")
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500543 return
544 self.do_deploy(
545 dry_run,
Brian Silvermanff485782014-06-18 19:59:09 -0700546 ('scp', '-o', 'Compression yes')
547 + tuple([os.path.join(from_dir, f) for f in to_download.decode('utf-8').split('\n')[:-1]])
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500548 + (('%s:%s' % (TARGET, TEMP_DIR)),))
549 if not dry_run:
Austin Schuhd3680052014-10-25 17:52:18 -0700550 mv_cmd = ['mv {TMPDIR}/* {TO_DIR} ']
551 if device == 'roboRIO':
552 mv_cmd.append('&& chmod u+s {TO_DIR}/starter_exe ')
553 mv_cmd.append('&& echo \'Done moving new executables into place\' ')
554 mv_cmd.append('&& bash -c \'sync && sync && sync\'')
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500555 subprocess.check_call(
556 ('ssh', TARGET,
Austin Schuhd3680052014-10-25 17:52:18 -0700557 ''.join(mv_cmd).format(TMPDIR=TEMP_DIR, TO_DIR=TARGET_DIR)))
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500558
Brian Silvermana4aff562014-05-02 17:43:50 -0700559 def build_env(self):
Brian Silvermane6bada62014-05-04 16:16:54 -0700560 OTHER_SYSROOT = '/opt/clang-3.5/'
561 SYMBOLIZER_PATH = OTHER_SYSROOT + 'bin/llvm-symbolizer'
Brian Silvermana4aff562014-05-02 17:43:50 -0700562 r = {}
Brian Silvermane6bada62014-05-04 16:16:54 -0700563 if self.compiler() == 'clang' or self.compiler() == 'gcc_4.8':
564 r['LD_LIBRARY_PATH'] = OTHER_SYSROOT + 'lib64'
565 if self.sanitizer() == 'address':
566 r['ASAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH
Brian Silverman452aaec2014-05-05 16:52:18 -0700567 r['ASAN_OPTIONS'] = \
Brian Silverman407ca822014-06-05 18:36:41 -0700568 'detect_leaks=1:check_initialization_order=1:strict_init_order=1' \
Brian Silverman415e65d2014-06-21 22:39:28 -0700569 ':detect_stack_use_after_return=1:detect_odr_violation=2' \
570 ':allow_user_segv_handler=1'
Brian Silvermane6bada62014-05-04 16:16:54 -0700571 elif self.sanitizer() == 'memory':
572 r['MSAN_SYMBOLIZER_PATH'] = SYMBOLIZER_PATH
573 elif self.sanitizer() == 'thread':
574 r['TSAN_OPTIONS'] = 'external_symbolizer_path=' + SYMBOLIZER_PATH
Brian Silvermand3fac732014-05-03 16:03:46 -0700575
576 r['CCACHE_COMPRESS'] = 'yes'
Brian Silverman452aaec2014-05-05 16:52:18 -0700577 r['CCACHE_DIR'] = os.path.abspath(os.path.join(aos_path(), '..', 'output',
578 'ccache_dir'))
Brian Silvermand3fac732014-05-03 16:03:46 -0700579 r['CCACHE_HASHDIR'] = 'yes'
Brian Silvermane6bada62014-05-04 16:16:54 -0700580 if self.compiler() == 'clang':
Brian Silvermand3fac732014-05-03 16:03:46 -0700581 # clang doesn't like being run directly on the preprocessed files.
582 r['CCACHE_CPP2'] = 'yes'
583 # Without this, ccache slows down because of the generated header files.
584 # The race condition that this opens up isn't a problem because the build
585 # system finishes modifying header files before compiling anything that
586 # uses them.
587 r['CCACHE_SLOPPINESS'] = 'include_file_mtime'
Brian Silvermand3fac732014-05-03 16:03:46 -0700588
Brian Silvermane6bada62014-05-04 16:16:54 -0700589 if self.architecture() == 'amd64':
Brian Silvermand3fac732014-05-03 16:03:46 -0700590 r['PATH'] = os.path.join(aos_path(), 'build', 'bin-ld.gold') + \
591 ':' + os.environ['PATH']
592
Brian Silvermana4aff562014-05-02 17:43:50 -0700593 return r
594
Brian Silverman47cd6f62014-05-03 10:35:52 -0700595 ARCHITECTURES = ('arm', 'amd64')
Brian Silvermanbae86d62014-09-14 01:05:31 -0400596 COMPILERS = ('clang', 'gcc', 'gcc_4.8', 'gcc_frc')
Brian Silverman47cd6f62014-05-03 10:35:52 -0700597 SANITIZERS = ('address', 'undefined', 'integer', 'memory', 'thread', 'none')
598 SANITIZER_TEST_WARNINGS = {
599 'memory': (True,
600"""We don't have all of the libraries instrumented which leads to lots of false
Brian Silvermanc3740c32014-05-04 12:42:47 -0700601 errors with msan (especially stdlibc++).
602 TODO(brians): Figure out a way to deal with it."""),
Brian Silverman47cd6f62014-05-03 10:35:52 -0700603 }
Brian Silvermana5301e32014-05-03 10:51:49 -0700604 PIE_SANITIZERS = ('memory', 'thread')
Brian Silvermana29ebf92014-04-23 13:08:49 -0500605
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500606 def __init__(self, is_test, is_deploy):
Brian Silverman452aaec2014-05-05 16:52:18 -0700607 super(PrimeProcessor, self).__init__()
Brian Silvermana29ebf92014-04-23 13:08:49 -0500608
609 platforms = []
610 for architecture in PrimeProcessor.ARCHITECTURES:
611 for compiler in PrimeProcessor.COMPILERS:
612 for debug in [True, False]:
Brian Silvermanbae86d62014-09-14 01:05:31 -0400613 if ((architecture == 'arm' and compiler == 'gcc_4.8') or
614 (architecture == 'amd64' and compiler == 'gcc_frc')):
Brian Silverman47cd6f62014-05-03 10:35:52 -0700615 # We don't have a compiler to use here.
616 continue
Brian Silvermana29ebf92014-04-23 13:08:49 -0500617 platforms.append(
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700618 PrimeProcessor.Platform(architecture, compiler, debug, 'none'))
619 for sanitizer in PrimeProcessor.SANITIZERS:
Brian Silverman47cd6f62014-05-03 10:35:52 -0700620 for compiler in ('gcc_4.8', 'clang'):
621 if compiler == 'gcc_4.8' and (sanitizer == 'undefined' or
622 sanitizer == 'integer' or
623 sanitizer == 'memory'):
624 # GCC 4.8 doesn't support these sanitizers.
625 continue
Brian Silvermane6bada62014-05-04 16:16:54 -0700626 if sanitizer == 'none':
627 # We already added sanitizer == 'none' above.
628 continue
629 platforms.append(
630 PrimeProcessor.Platform('amd64', compiler, True, sanitizer))
631 self.__platforms = frozenset(platforms)
632
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500633 if is_test:
Brian Silvermane6bada62014-05-04 16:16:54 -0700634 default_platforms = self.select_platforms(architecture='amd64',
635 debug=True)
Brian Silvermanc3740c32014-05-04 12:42:47 -0700636 for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items():
637 if warning[0]:
Brian Silvermane6bada62014-05-04 16:16:54 -0700638 default_platforms -= self.select_platforms(sanitizer=sanitizer)
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500639 elif is_deploy:
Austin Schuhd3680052014-10-25 17:52:18 -0700640 if get_ip_base() == '10.99.71':
641 compiler = 'gcc_frc'
642 else:
643 compiler = 'clang'
Brian Silvermane6bada62014-05-04 16:16:54 -0700644 default_platforms = self.select_platforms(architecture='arm',
Austin Schuhd3680052014-10-25 17:52:18 -0700645 compiler=compiler,
Brian Silvermane6bada62014-05-04 16:16:54 -0700646 debug=False)
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500647 else:
Brian Silvermane6bada62014-05-04 16:16:54 -0700648 default_platforms = self.select_platforms(debug=False)
649 self.__default_platforms = frozenset(default_platforms)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500650
Brian Silvermane6bada62014-05-04 16:16:54 -0700651 def platforms(self):
652 return self.__platforms
653 def default_platforms(self):
654 return self.__default_platforms
655
656 def download_externals(self, platforms):
657 to_download = set()
658 for architecture in PrimeProcessor.ARCHITECTURES:
Brian Silverman99895b92014-09-14 01:01:15 -0400659 pie_sanitizers = set()
Brian Silvermane6bada62014-05-04 16:16:54 -0700660 for sanitizer in PrimeProcessor.PIE_SANITIZERS:
Brian Silverman99895b92014-09-14 01:01:15 -0400661 pie_sanitizers.update(self.select_platforms(architecture=architecture,
662 sanitizer=sanitizer))
663 if platforms & pie_sanitizers:
664 to_download.add(architecture + '-fPIE')
665
Brian Silvermanbae86d62014-09-14 01:05:31 -0400666 frc_platforms = self.select_platforms(architecture=architecture,
667 compiler='gcc_frc')
668 if platforms & frc_platforms:
669 to_download.add(architecture + '_frc')
670
671 if platforms & (self.platforms() - pie_sanitizers - frc_platforms):
Brian Silverman99895b92014-09-14 01:01:15 -0400672 to_download.add(architecture)
673
Brian Silvermane6bada62014-05-04 16:16:54 -0700674 for download_target in to_download:
675 call_download_externals(download_target)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500676
Brian Silverman452aaec2014-05-05 16:52:18 -0700677 def parse_platforms(self, platform_string):
678 if platform_string is None:
Brian Silvermane6bada62014-05-04 16:16:54 -0700679 return self.default_platforms()
680 r = self.default_platforms()
Brian Silverman452aaec2014-05-05 16:52:18 -0700681 for part in platform_string.split(','):
Brian Silverman1867aae2014-05-05 17:16:34 -0700682 if part == 'all':
683 r = self.platforms()
684 elif part[0] == '+':
Brian Silvermana29ebf92014-04-23 13:08:49 -0500685 r = r | self.select_platforms_string(part[1:])
686 elif part[0] == '-':
687 r = r - self.select_platforms_string(part[1:])
688 elif part[0] == '=':
689 r = self.select_platforms_string(part[1:])
690 else:
Brian Silverman7cd5ad42014-04-27 08:11:30 -0500691 selected = self.select_platforms_string(part)
Brian Silvermane6bada62014-05-04 16:16:54 -0700692 r = r - (self.platforms() - selected)
Brian Silverman7cd5ad42014-04-27 08:11:30 -0500693 if not r:
694 r = selected
Brian Silvermana29ebf92014-04-23 13:08:49 -0500695 return r
696
Brian Silverman452aaec2014-05-05 16:52:18 -0700697 def select_platforms(self, architecture=None, compiler=None, debug=None,
698 sanitizer=None):
Brian Silvermana29ebf92014-04-23 13:08:49 -0500699 r = []
Brian Silvermane6bada62014-05-04 16:16:54 -0700700 for platform in self.platforms():
701 if architecture is None or platform.architecture() == architecture:
702 if compiler is None or platform.compiler() == compiler:
703 if debug is None or platform.debug() == debug:
704 if sanitizer is None or platform.sanitizer() == sanitizer:
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700705 r.append(platform)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500706 return set(r)
707
Brian Silverman452aaec2014-05-05 16:52:18 -0700708 def select_platforms_string(self, platforms_string):
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700709 architecture, compiler, debug, sanitizer = None, None, None, None
Brian Silverman452aaec2014-05-05 16:52:18 -0700710 for part in platforms_string.split('-'):
Brian Silvermana29ebf92014-04-23 13:08:49 -0500711 if part in PrimeProcessor.ARCHITECTURES:
712 architecture = part
713 elif part in PrimeProcessor.COMPILERS:
714 compiler = part
715 elif part in ['debug', 'dbg']:
716 debug = True
717 elif part in ['release', 'nodebug', 'ndb']:
718 debug = False
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700719 elif part in PrimeProcessor.SANITIZERS:
720 sanitizer = part
Brian Silverman415e65d2014-06-21 22:39:28 -0700721 elif part == 'all':
722 architecture = compiler = debug = sanitizer = None
Brian Silvermana29ebf92014-04-23 13:08:49 -0500723 else:
Brian Silverman452aaec2014-05-05 16:52:18 -0700724 raise Processor.UnknownPlatform(
Brian Silvermandf5348a2014-06-12 23:25:08 -0700725 '"%s" not recognized as a platform string component.' % part)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500726 return self.select_platforms(
727 architecture=architecture,
728 compiler=compiler,
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700729 debug=debug,
730 sanitizer=sanitizer)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500731
Brian Silverman9b7a6842014-05-05 16:19:11 -0700732 def check_installed(self, platforms, is_deploy):
733 packages = set(('lzip', 'm4', 'realpath'))
734 packages.add('ruby')
735 # clang-format from here gets used for all versions.
736 packages.add('clang-3.5')
737 packages.add('arm-eabi-gcc')
738 for platform in platforms:
739 if platform.architecture() == 'arm':
740 packages.add('gcc-4.7-arm-linux-gnueabihf')
741 packages.add('g++-4.7-arm-linux-gnueabihf')
742 if platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8':
743 packages.add('clang-3.5')
Brian Silverman1867aae2014-05-05 17:16:34 -0700744 if platform.compiler() == 'gcc_4.8':
745 packages.add('libcloog-isl3:amd64')
Brian Silverman9b7a6842014-05-05 16:19:11 -0700746 if is_deploy:
747 packages.add('openssh-client')
748 if platform.compiler() == 'gcc' and platform.architecture() == 'amd64':
749 packages.add('gcc-4.7')
750 packages.add('g++-4.7')
Brian Silvermanbae86d62014-09-14 01:05:31 -0400751 elif platform.compiler() == 'gcc_frc':
752 packages.add('gcc-4.9-arm-frc-linux-gnueabi')
753 packages.add('g++-4.9-arm-frc-linux-gnueabi')
Brian Silverman9b7a6842014-05-05 16:19:11 -0700754
755 self.do_check_installed(tuple(packages))
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700756
Brian Silverman6bca4722014-05-20 17:02:49 -0700757def strsignal(num):
758 # It ends up with SIGIOT instead otherwise, which is weird.
759 if num == signal.SIGABRT:
760 return 'SIGABRT'
761 # SIGCLD is a weird way to spell it.
762 if num == signal.SIGCHLD:
763 return 'SIGCHLD'
764
765 SIGNALS_TO_NAMES = dict((getattr(signal, n), n)
766 for n in dir(signal) if n.startswith('SIG')
767 and '_' not in n)
768 return SIGNALS_TO_NAMES.get(num, 'Unknown signal %d' % num)
769
Brian Silvermana29ebf92014-04-23 13:08:49 -0500770def main():
Brian Silvermandf5348a2014-06-12 23:25:08 -0700771 sys.argv.pop(0)
772 exec_name = sys.argv.pop(0)
773 def print_help(exit_status=None, message=None):
774 if message:
775 print(message)
776 sys.stdout.write(
Brian Silvermanb9e89602014-06-27 14:21:08 -0500777"""Usage: {name} [-j n] [action] [-n] [platform] [target|extra_flag]...
Brian Silvermandf5348a2014-06-12 23:25:08 -0700778Arguments:
779 -j, --jobs Explicitly specify how many jobs to run at a time.
780 Defaults to the number of processors + 2.
781 -n, --dry-run Don't actually do whatever.
782 Currently only meaningful for deploy.
783 action What to do. Defaults to build.
784 build: Build the code.
785 clean: Remove all the built output.
786 tests: Build and then run tests.
787 deploy: Build and then download.
788 platform What variants of the code to build.
789 Defaults to something reasonable.
790 See below for details.
791 target... Which targets to build/test/etc.
792 Defaults to everything.
Brian Silvermanb9e89602014-06-27 14:21:08 -0500793 extra_flag... Extra flags associated with the targets.
794 --gtest_*: Arguments to pass on to tests.
Brian Silvermana29ebf92014-04-23 13:08:49 -0500795
Brian Silvermandf5348a2014-06-12 23:25:08 -0700796Specifying targets:
797 Targets are combinations of architecture, compiler, and debug flags. Which
798 ones actually get run is built up as a set. It defaults to something
799 reasonable for the action (specified below).
800 The platform specification (the argument given to this script) is a comma-
801 separated sequence of hyphen-separated platforms, each with an optional
802 prefix.
803 Each selector (the things separated by commas) selects all of the platforms
804 which match all of its components. Its effect on the set of current platforms
805 depends on the prefix character.
806 Here are the prefix characters:
807 + Adds the selected platforms.
808 - Removes the selected platforms.
809 = Sets the current set to the selected platforms.
810 [none] Removes all non-selected platforms.
811 If this makes the current set empty, acts like =.
812 There is also the special psuedo-platform "all" which selects all platforms.
813 All of the available platforms:
814 {all_platforms}
815 Default platforms for deploying:
816 {deploy_platforms}
817 Default platforms for testing:
818 {test_platforms}
819 Default platforms for everything else:
820 {default_platforms}
Brian Silvermana29ebf92014-04-23 13:08:49 -0500821
Brian Silvermandf5348a2014-06-12 23:25:08 -0700822Examples of specifying targets:
823 build everything: "all"
824 only build things with clang: "clang"
825 build everything that uses GCC 4.8 (not just the defaults): "=gcc_4.8"
826 build all of the arm targets that use clang: "clang-arm" or "arm-clang"
827""".format(
828 name=exec_name,
829 all_platforms=str_platforms(PrimeProcessor(False, False).platforms()),
830 deploy_platforms=str_platforms(PrimeProcessor(False, True).default_platforms()),
831 test_platforms=str_platforms(PrimeProcessor(True, False).default_platforms()),
832 default_platforms=str_platforms(PrimeProcessor(False, False).default_platforms()),
833 ))
834 if exit_status is not None:
835 sys.exit(exit_status)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500836
Brian Silvermandf5348a2014-06-12 23:25:08 -0700837 def sort_platforms(platforms):
838 return sorted(
839 platforms, key=lambda platform: (-platform.priority(), str(platform)))
Brian Silvermana29ebf92014-04-23 13:08:49 -0500840
Brian Silvermandf5348a2014-06-12 23:25:08 -0700841 def str_platforms(platforms):
842 r = []
843 for platform in sort_platforms(platforms):
844 r.append(str(platform))
845 if len(r) > 1:
846 r[-1] = 'and ' + r[-1]
847 return ', '.join(r)
848
849 class Arguments(object):
850 def __init__(self):
851 self.jobs = os.sysconf('SC_NPROCESSORS_ONLN') + 2
852 self.action_name = 'build'
853 self.dry_run = False
854 self.targets = []
855 self.platform = None
Brian Silvermanb9e89602014-06-27 14:21:08 -0500856 self.extra_flags = []
Brian Silvermana29ebf92014-04-23 13:08:49 -0500857
Brian Silvermandf5348a2014-06-12 23:25:08 -0700858 args = Arguments()
Brian Silvermana29ebf92014-04-23 13:08:49 -0500859
Brian Silvermandf5348a2014-06-12 23:25:08 -0700860 if len(sys.argv) < 2:
861 print_help(1, 'Not enough arguments')
862 args.processor = sys.argv.pop(0)
863 args.main_gyp = sys.argv.pop(0)
864 VALID_ACTIONS = ['build', 'clean', 'deploy', 'tests']
865 while sys.argv:
866 arg = sys.argv.pop(0)
867 if arg == '-j' or arg == '--jobs':
868 args.jobs = int(sys.argv.pop(0))
869 continue
870 if arg in VALID_ACTIONS:
871 args.action_name = arg
872 continue
873 if arg == '-n' or arg == '--dry-run':
874 if args.action_name != 'deploy':
875 print_help(1, '--dry-run is only valid for deploy')
876 args.dry_run = True
877 continue
878 if arg == '-h' or arg == '--help':
879 print_help(0)
Brian Silvermanb9e89602014-06-27 14:21:08 -0500880 if re.match('^--gtest_.*$', arg):
881 if args.action_name == 'tests':
882 args.extra_flags.append(arg)
883 continue
884 else:
885 print_help(1, '--gtest_* is only valid for tests')
Brian Silvermandf5348a2014-06-12 23:25:08 -0700886 if args.platform:
887 args.targets.append(arg)
888 else:
889 args.platform = arg
Brian Silvermana29ebf92014-04-23 13:08:49 -0500890
891 if args.processor == 'crio':
892 processor = CRIOProcessor()
893 elif args.processor == 'prime':
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500894 processor = PrimeProcessor(args.action_name == 'tests',
895 args.action_name == 'deploy')
Brian Silvermana29ebf92014-04-23 13:08:49 -0500896 else:
Brian Silvermandf5348a2014-06-12 23:25:08 -0700897 print_help(1, message='Unknown processor "%s".' % args.processor)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500898
Brian Silvermana29ebf92014-04-23 13:08:49 -0500899 unknown_platform_error = None
900 try:
Brian Silvermandf5348a2014-06-12 23:25:08 -0700901 platforms = processor.parse_platforms(args.platform)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500902 except Processor.UnknownPlatform as e:
903 unknown_platform_error = e.message
Brian Silvermandf5348a2014-06-12 23:25:08 -0700904 args.targets.insert(0, args.platform)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500905 platforms = processor.parse_platforms(None)
906 if not platforms:
Brian Silvermandf5348a2014-06-12 23:25:08 -0700907 print_help(1, 'No platforms selected')
Brian Silvermana29ebf92014-04-23 13:08:49 -0500908
Brian Silverman9b7a6842014-05-05 16:19:11 -0700909 processor.check_installed(platforms, args.action_name == 'deploy')
Brian Silvermane6bada62014-05-04 16:16:54 -0700910 processor.download_externals(platforms)
Brian Silvermana29ebf92014-04-23 13:08:49 -0500911
912 class ToolsConfig(object):
913 def __init__(self):
Brian Silvermanbe6cfe22014-04-27 08:06:27 -0500914 self.variables = {'AOS': aos_path()}
915 with open(os.path.join(aos_path(), 'build', 'tools_config'), 'r') as f:
Brian Silvermana29ebf92014-04-23 13:08:49 -0500916 for line in f:
917 if line[0] == '#':
918 pass
919 elif line.isspace():
920 pass
921 else:
922 new_name, new_value = line.rstrip().split('=')
923 for name, value in self.variables.items():
924 new_value = new_value.replace('${%s}' % name, value)
925 self.variables[new_name] = new_value
926 def __getitem__(self, key):
927 return self.variables[key]
928
929 tools_config = ToolsConfig()
930
931 def handle_clean_error(function, path, excinfo):
Brian Silverman452aaec2014-05-05 16:52:18 -0700932 _, _ = function, path
Brian Silvermana29ebf92014-04-23 13:08:49 -0500933 if issubclass(OSError, excinfo[0]):
934 if excinfo[1].errno == errno.ENOENT:
935 # Who cares if the file we're deleting isn't there?
936 return
937 raise excinfo[1]
938
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700939 def need_to_run_gyp(platform):
Brian Silvermane6bada62014-05-04 16:16:54 -0700940 """Determines if we need to run gyp again or not.
941
942 The generated build files are supposed to re-run gyp again themselves, but
943 that doesn't work (or at least it used to not) and we sometimes want to
944 modify the results anyways.
945
946 Args:
947 platform: The platform to check for.
948 """
Brian Silvermanf0d3c782014-05-02 23:56:32 -0700949 if not os.path.exists(platform.build_ninja()):
950 return True
Brian Silvermane6bada62014-05-04 16:16:54 -0700951 if os.path.getmtime(__file__) > os.path.getmtime(platform.build_ninja()):
952 return True
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700953 dirs = os.listdir(os.path.join(aos_path(), '..'))
Brian Silvermana4aff562014-05-02 17:43:50 -0700954 # Looking through these folders takes a long time and isn't useful.
Brian Silverman452aaec2014-05-05 16:52:18 -0700955 if dirs.count('output'):
956 dirs.remove('output')
957 if dirs.count('.git'):
958 dirs.remove('.git')
Brian Silvermana4aff562014-05-02 17:43:50 -0700959 return not not subprocess.check_output(
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700960 ('find',) + tuple(os.path.join(aos_path(), '..', d) for d in dirs)
Brian Silverman452aaec2014-05-05 16:52:18 -0700961 + ('-newer', platform.build_ninja(),
962 '(', '-name', '*.gyp', '-or', '-name', '*.gypi', ')'),
Brian Silvermana4aff562014-05-02 17:43:50 -0700963 stdin=open(os.devnull, 'r'))
964
965 def env(platform):
Brian Silvermane6bada62014-05-04 16:16:54 -0700966 """Makes sure we pass through important environmental variables.
967
968 Returns:
969 An environment suitable for passing to subprocess.Popen and friends.
970 """
Brian Silvermana4aff562014-05-02 17:43:50 -0700971 build_env = dict(platform.build_env())
Brian Silvermand3fac732014-05-03 16:03:46 -0700972 if not 'TERM' in build_env:
973 build_env['TERM'] = os.environ['TERM']
974 if not 'PATH' in build_env:
975 build_env['PATH'] = os.environ['PATH']
Brian Silvermana4aff562014-05-02 17:43:50 -0700976 return build_env
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700977
Brian Silvermandf5348a2014-06-12 23:25:08 -0700978 sorted_platforms = sort_platforms(platforms)
979 user_output('Building %s...' % str_platforms(sorted_platforms))
Brian Silverman47cd6f62014-05-03 10:35:52 -0700980
981 if args.action_name == 'tests':
982 for sanitizer, warning in PrimeProcessor.SANITIZER_TEST_WARNINGS.items():
983 warned_about = platforms & processor.select_platforms(sanitizer=sanitizer)
984 if warned_about:
985 user_output(warning[1])
986 if warning[0]:
Brian Silvermane6bada62014-05-04 16:16:54 -0700987 # TODO(brians): Add a --force flag or something to override this?
Brian Silverman47cd6f62014-05-03 10:35:52 -0700988 user_output('Refusing to run tests for sanitizer %s.' % sanitizer)
989 exit(1)
990
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700991 num = 1
Brian Silvermanbd380fd2014-05-13 16:55:24 -0700992 for platform in sorted_platforms:
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -0700993 user_output('Building %s (%d/%d)...' % (platform, num, len(platforms)))
Brian Silvermana29ebf92014-04-23 13:08:49 -0500994 if args.action_name == 'clean':
995 shutil.rmtree(platform.outdir(), onerror=handle_clean_error)
996 else:
997 if need_to_run_gyp(platform):
Brian Silvermana9b1e5c2014-04-30 18:08:04 -0700998 user_output('Running gyp...')
Brian Silvermana29ebf92014-04-23 13:08:49 -0500999 gyp = subprocess.Popen(
1000 (tools_config['GYP'],
1001 '--check',
Brian Silvermanbe6cfe22014-04-27 08:06:27 -05001002 '--depth=%s' % os.path.join(aos_path(), '..'),
Brian Silvermana29ebf92014-04-23 13:08:49 -05001003 '--no-circular-check',
1004 '-f', 'ninja',
Brian Silvermanbe6cfe22014-04-27 08:06:27 -05001005 '-I%s' % os.path.join(aos_path(), 'build', 'aos.gypi'),
Brian Silvermana29ebf92014-04-23 13:08:49 -05001006 '-I/dev/stdin', '-Goutput_dir=output',
1007 '-DOS=%s' % platform.os(),
1008 '-DPLATFORM=%s' % platform.gyp_platform(),
Brian Silvermane6bada62014-05-04 16:16:54 -07001009 '-DARCHITECTURE=%s' % platform.architecture(),
1010 '-DCOMPILER=%s' % platform.compiler().split('_')[0],
1011 '-DFULL_COMPILER=%s' % platform.compiler(),
1012 '-DDEBUG=%s' % ('yes' if platform.debug() else 'no'),
1013 '-DSANITIZER=%s' % platform.sanitizer(),
Brian Silverman99895b92014-09-14 01:01:15 -04001014 '-DEXTERNALS_EXTRA=%s' %
Brian Silverman452aaec2014-05-05 16:52:18 -07001015 ('-fPIE' if platform.sanitizer() in PrimeProcessor.PIE_SANITIZERS
Brian Silvermanbae86d62014-09-14 01:05:31 -04001016 else ('_frc' if platform.compiler() == 'gcc_frc' else ''))) +
Brian Silvermanb3d50542014-04-23 14:28:55 -05001017 processor.extra_gyp_flags() + (args.main_gyp,),
Brian Silvermana29ebf92014-04-23 13:08:49 -05001018 stdin=subprocess.PIPE)
1019 gyp.communicate(("""
1020{
1021 'target_defaults': {
1022 'configurations': {
1023 '%s': {}
1024 }
1025 }
1026}""" % platform.outname()).encode())
1027 if gyp.returncode:
Brian Silvermana9b1e5c2014-04-30 18:08:04 -07001028 user_output("Running gyp failed!")
Brian Silvermana29ebf92014-04-23 13:08:49 -05001029 exit(1)
Brian Silvermane6bada62014-05-04 16:16:54 -07001030 processor.modify_ninja_file(platform.build_ninja())
Brian Silverman47cd6f62014-05-03 10:35:52 -07001031 user_output('Done running gyp')
Brian Silvermana29ebf92014-04-23 13:08:49 -05001032 else:
Brian Silverman47cd6f62014-05-03 10:35:52 -07001033 user_output("Not running gyp")
Brian Silvermana29ebf92014-04-23 13:08:49 -05001034
1035 try:
Brian Silvermanc3740c32014-05-04 12:42:47 -07001036 call = (tools_config['NINJA'],
Brian Silvermandf5348a2014-06-12 23:25:08 -07001037 '-C', platform.outdir()) + tuple(args.targets)
Brian Silvermanc3740c32014-05-04 12:42:47 -07001038 if args.jobs:
1039 call += ('-j', str(args.jobs))
1040 subprocess.check_call(call,
Brian Silverman452aaec2014-05-05 16:52:18 -07001041 stdin=open(os.devnull, 'r'),
1042 env=env(platform))
Brian Silvermana29ebf92014-04-23 13:08:49 -05001043 except subprocess.CalledProcessError as e:
1044 if unknown_platform_error is not None:
Brian Silvermana9b1e5c2014-04-30 18:08:04 -07001045 user_output(unknown_platform_error)
Brian Silvermana29ebf92014-04-23 13:08:49 -05001046 raise e
1047
Brian Silvermanbe6cfe22014-04-27 08:06:27 -05001048 if args.action_name == 'deploy':
1049 platform.deploy(args.dry_run)
Brian Silvermane48c09a2014-04-30 18:04:58 -07001050 elif args.action_name == 'tests':
1051 dirname = os.path.join(platform.outdir(), 'tests')
Brian Silvermanc3740c32014-05-04 12:42:47 -07001052 done_queue = queue.Queue()
1053 running = []
Brian Silvermandf5348a2014-06-12 23:25:08 -07001054 test_start_semaphore = threading.Semaphore(args.jobs)
1055 if args.targets:
Brian Silvermanc3740c32014-05-04 12:42:47 -07001056 to_run = []
Brian Silvermandf5348a2014-06-12 23:25:08 -07001057 for target in args.targets:
Brian Silvermanc3740c32014-05-04 12:42:47 -07001058 if target.endswith('_test'):
1059 to_run.append(target)
1060 else:
1061 to_run = os.listdir(dirname)
1062 for f in to_run:
Brian Silvermanb9e89602014-06-27 14:21:08 -05001063 thread = TestThread(os.path.join(dirname, f), tuple(args.extra_flags),
1064 env(platform), done_queue,
Brian Silvermanc3740c32014-05-04 12:42:47 -07001065 test_start_semaphore)
1066 running.append(thread)
1067 thread.start()
1068 try:
1069 while running:
1070 done = done_queue.get()
1071 running.remove(done)
1072 with test_output_lock:
1073 test_output('Output from test %s:' % done.name)
Brian Silverman730bb012014-06-08 13:05:20 -07001074 try:
1075 while True:
1076 line = done.output.get(False)
1077 if not sys.stdout.isatty():
1078 # Remove color escape codes.
1079 line = re.sub(r'\x1B\[[0-9;]*[a-zA-Z]', '', line)
1080 sys.stdout.write(line)
1081 except queue.Empty:
1082 pass
1083 if not done.returncode:
1084 test_output('Test %s succeeded' % done.name)
1085 else:
1086 if done.returncode < 0:
1087 sig = -done.returncode
1088 test_output('Test %s was killed by signal %d (%s)' % \
1089 (done.name, sig, strsignal(sig)))
1090 elif done.returncode != 1:
1091 test_output('Test %s exited with %d' % \
1092 (done.name, done.returncode))
Brian Silvermanf2bbe092014-05-13 16:55:03 -07001093 else:
Brian Silverman730bb012014-06-08 13:05:20 -07001094 test_output('Test %s failed' % done.name)
1095 user_output('Aborting because of test failure for %s.' % \
1096 platform)
1097 exit(1)
Brian Silvermanc3740c32014-05-04 12:42:47 -07001098 finally:
1099 if running:
1100 test_output('Killing other tests...')
Brian Silvermanbf0e1db2014-05-10 22:13:15 -07001101# Stop all of them before killing processes because otherwise stopping some of
1102# them tends to let other ones that are waiting to start go.
1103 for thread in running:
Brian Silvermana5826582014-06-03 19:46:35 -07001104 test_output('\tKilling %s' % thread.name)
Brian Silvermanbf0e1db2014-05-10 22:13:15 -07001105 thread.stop()
Brian Silvermanc3740c32014-05-04 12:42:47 -07001106 for thread in running:
1107 thread.terminate_process()
1108 to_remove = []
1109 for thread in running:
1110 thread.join(5)
1111 if not thread.is_alive():
Brian Silverman452aaec2014-05-05 16:52:18 -07001112 to_remove.append(thread)
1113 for thread in to_remove:
1114 running.remove(thread)
Brian Silvermanc3740c32014-05-04 12:42:47 -07001115 for thread in running:
1116 test_output(
1117 'Test %s did not terminate. Killing it.' % thread.name)
1118 thread.kill_process()
1119 thread.join()
1120 test_output('Done killing other tests')
Brian Silvermanbe6cfe22014-04-27 08:06:27 -05001121
Brian Silvermanc2d8e5a2014-05-01 18:33:12 -07001122 user_output('Done building %s (%d/%d)' % (platform, num, len(platforms)))
1123 num += 1
Brian Silvermana29ebf92014-04-23 13:08:49 -05001124
1125if __name__ == '__main__':
1126 main()