got the python-based build scripts working (for amd64)
diff --git a/aos/build/aos.gypi b/aos/build/aos.gypi
index c45c041..c6fc79d 100644
--- a/aos/build/aos.gypi
+++ b/aos/build/aos.gypi
@@ -15,29 +15,6 @@
'so_dir': '<(PRODUCT_DIR)/lib',
# the directory that executables that depend on <(EXTERNALS):gtest get put into
'test_dir': '<(PRODUCT_DIR)/tests',
- 'variables': {
- 'conditions': [
- ['PLATFORM=="linux"', {
- 'compiler': 'gcc',
- 'aos_architecture': 'arm',
- },
- ], ['PLATFORM=="linux-amd64-clang"', {
- 'compiler': 'clang',
- 'aos_architecture': 'amd64',
- },
- ], ['PLATFORM=="linux-clang"', {
- 'compiler': 'gcc',
- 'aos_architecture': 'arm',
- },
- ], ['PLATFORM=="linux-amd64"', {
- 'compiler': 'gcc',
- 'aos_architecture': 'amd64',
- },
- ],
- ],
- },
- 'aos_architecture%': '<(aos_architecture)',
- 'compiler%': '<(compiler)',
},
'conditions': [
['OS=="crio"', {
@@ -46,13 +23,13 @@
['CXX', '<!(readlink -f <(AOS)/build/crio_cxx)'],
],
}
- ], ['PLATFORM=="linux"', {
+ ], ['PLATFORM=="linux-arm-gcc"', {
'make_global_settings': [
['CC', '<!(which arm-linux-gnueabihf-gcc-4.7)'],
['CXX', '<!(which arm-linux-gnueabihf-g++-4.7)'],
],
},
- ], ['PLATFORM=="linux-clang"', {
+ ], ['PLATFORM=="linux-arm-clang"', {
# TODO(brians): Make this one actually work.
},
], ['PLATFORM=="linux-amd64-clang"', {
@@ -61,7 +38,7 @@
['CXX', '<!(which clang++)'],
],
},
- ], ['PLATFORM=="linux-amd64"', {
+ ], ['PLATFORM=="linux-amd64-gcc"', {
},
],
],
@@ -220,12 +197,12 @@
'-lrt',
],
'conditions': [
- ['compiler=="gcc"', {
+ ['COMPILER=="gcc"', {
'cflags': [
'-Wunused-local-typedefs',
],
},
- ], ['compiler=="clang"', {
+ ], ['COMPILER=="clang"', {
'cflags': [
'-fcolor-diagnostics',
],
diff --git a/aos/build/build.py b/aos/build/build.py
new file mode 100755
index 0000000..8b7790f
--- /dev/null
+++ b/aos/build/build.py
@@ -0,0 +1,314 @@
+#!/usr/bin/python3
+
+import argparse
+import sys
+import subprocess
+import re
+import os
+import os.path
+import string
+import shutil
+import errno
+
+class Processor(object):
+ class UnknownPlatform(Exception):
+ def __init__(self, message):
+ self.message = message
+
+ def aos_path():
+ return os.path.join(os.path.dirname(__file__), '..')
+
+class CRIOProcessor(Processor):
+ def __init__(self):
+ super(CRIOProcessor, self).__init__()
+
+ if 'WIND_BASE' in os.environ:
+ self.wind_base = os.environ['WIND_BASE']
+ else:
+ self.wind_base = '/usr/local/powerpc-wrs-vxworks/wind_base'
+
+ def parse_platforms(self, string):
+ if string is not None and string != 'crio':
+ raise Processor.UnknownPlatform('Unknown cRIO platform "%s"!' % string, file=sys.stderr)
+ return CRIOPlatform()
+
+ def build_env(self):
+ return {'WIND_BASE': self.wind_base}
+ def is_crio(self): return True
+
+class PrimeProcessor(Processor):
+ class Platform(object):
+ def __init__(self, architecture, compiler, debug):
+ self.architecture = architecture
+ self.compiler = compiler
+ self.debug = debug
+
+ def __repr__(self):
+ return 'PrimeProcessor.Platform(architecture=%s, compiler=%s, debug=%s)' \
+ % (self.architecture, self.compiler, self.debug)
+ def __str__(self):
+ return '%s-%s%s' % (self.architecture, self.compiler,
+ '-debug' if self.debug else '')
+
+ def os(self):
+ return 'linux'
+ def gyp_platform(self):
+ return '%s-%s-%s' % (self.os(), self.architecture, self.compiler)
+
+ def outdir(self):
+ return os.path.join(
+ Processor.aos_path(), '..', 'output', self.outname())
+ def outname(self):
+ return str(self)
+ def build_ninja(self):
+ return os.path.join(self.outdir(), 'build.ninja')
+
+ ARCHITECTURES = ['arm', 'amd64']
+ COMPILERS = ['clang', 'gcc']
+
+ def __init__(self):
+ super(Processor, self).__init__()
+
+ platforms = []
+ for architecture in PrimeProcessor.ARCHITECTURES:
+ for compiler in PrimeProcessor.COMPILERS:
+ for debug in [True, False]:
+ platforms.append(
+ PrimeProcessor.Platform(architecture, compiler, debug))
+ self.platforms = frozenset(platforms)
+ self.default_platforms = self.select_platforms(debug=False)
+
+ def build_env(self):
+ return {}
+ def is_crio(self): return False
+
+ def parse_platforms(self, string):
+ if string is None:
+ return self.default_platforms
+ r = self.default_platforms
+ for part in string.split(','):
+ if part[0] == '+':
+ r = r | self.select_platforms_string(part[1:])
+ elif part[0] == '-':
+ r = r - self.select_platforms_string(part[1:])
+ elif part[0] == '=':
+ r = self.select_platforms_string(part[1:])
+ else:
+ r = r - (self.platforms - self.select_platforms_string(part))
+ return r
+
+ def select_platforms(self, architecture=None, compiler=None, debug=None):
+ r = []
+ for platform in self.platforms:
+ if architecture is None or platform.architecture == architecture:
+ if compiler is None or platform.compiler == compiler:
+ if debug is None or platform.debug == debug:
+ r.append(platform)
+ return set(r)
+
+ def select_platforms_string(self, string):
+ r = []
+ architecture, compiler, debug = None, None, None
+ for part in string.split('-'):
+ if part in PrimeProcessor.ARCHITECTURES:
+ architecture = part
+ elif part in PrimeProcessor.COMPILERS:
+ compiler = part
+ elif part in ['debug', 'dbg']:
+ debug = True
+ elif part in ['release', 'nodebug', 'ndb']:
+ debug = False
+ else:
+ raise Processor.UnknownPlatform('Unknown platform string component "%s".' % part)
+ return self.select_platforms(
+ architecture=architecture,
+ compiler=compiler,
+ debug=debug)
+
+def main():
+ class TryParsingAgain(Exception):
+ pass
+
+ class TryAgainArgumentParser(argparse.ArgumentParser):
+ def __init__(self, **kwargs):
+ super(TryAgainArgumentParser, self).__init__(**kwargs)
+
+ def error(self, message):
+ raise TryParsingAgain
+
+ def SetUpParser(parser, args):
+ def AddBuildArgs(parser):
+ parser.add_argument(
+ 'target',
+ help='target to build',
+ nargs='*')
+ def AddCommonArgs(parser):
+ parser.add_argument(
+ 'platforms',
+ help='platform(s) to act on',
+ nargs='?')
+
+ parser.add_argument('--processor', required=True, help='prime or crio')
+ parser.add_argument('--main_gyp', required=True, help='main .gyp file')
+ subparsers = parser.add_subparsers(dest='action_name')
+
+ build_parser = subparsers.add_parser(
+ 'build',
+ help='build the code (default)')
+ AddCommonArgs(build_parser)
+ AddBuildArgs(build_parser)
+
+ clean_parser = subparsers.add_parser(
+ 'clean',
+ help='remove all output directories')
+ AddCommonArgs(clean_parser)
+
+ deploy_parser = subparsers.add_parser(
+ 'deploy',
+ help='build and download the code')
+ AddCommonArgs(deploy_parser)
+ AddBuildArgs(deploy_parser)
+ deploy_parser.add_argument(
+ '-n', '--dry-run',
+ help="don't actually download anything",
+ action='store_true')
+
+ return parser.parse_args(args)
+
+ try:
+ parser = TryAgainArgumentParser()
+ args = SetUpParser(parser, sys.argv[1:])
+ except TryParsingAgain:
+ parser = argparse.ArgumentParser()
+ REQUIRED_ARGS_END = 5
+ args = SetUpParser(parser, sys.argv[1:REQUIRED_ARGS_END] + ['build'] +
+ sys.argv[(REQUIRED_ARGS_END):])
+
+ if args.processor == 'crio':
+ processor = CRIOProcessor()
+ elif args.processor == 'prime':
+ processor = PrimeProcessor()
+ else:
+ parser.exit(status=1, message='Unknown processor "%s".' % args.processor)
+
+ if 'target' in args:
+ targets = args.target[:]
+ else:
+ targets = []
+ unknown_platform_error = None
+ try:
+ platforms = processor.parse_platforms(args.platforms)
+ except Processor.UnknownPlatform as e:
+ unknown_platform_error = e.message
+ targets.append(args.platforms)
+ platforms = processor.parse_platforms(None)
+ if not platforms:
+ print("No platforms selected!", file=sys.stderr)
+ exit(1)
+
+ def download_externals(argument):
+ subprocess.check_call(
+ (os.path.join(Processor.aos_path(), 'build', 'download_externals.sh'),
+ argument),
+ stdin=open('/dev/null', 'r'))
+
+ if processor.is_crio():
+ download_externals('crio')
+ else:
+ for architecture in PrimeProcessor.ARCHITECTURES:
+ if platforms & processor.select_platforms(architecture=architecture):
+ download_externals(architecture)
+
+ class ToolsConfig(object):
+ def __init__(self):
+ self.variables = {'AOS': Processor.aos_path()}
+ with open(os.path.join(Processor.aos_path(), 'build', 'tools_config'), 'r') as f:
+ for line in f:
+ if line[0] == '#':
+ pass
+ elif line.isspace():
+ pass
+ else:
+ new_name, new_value = line.rstrip().split('=')
+ for name, value in self.variables.items():
+ new_value = new_value.replace('${%s}' % name, value)
+ self.variables[new_name] = new_value
+ def __getitem__(self, key):
+ return self.variables[key]
+
+ tools_config = ToolsConfig()
+
+ def handle_clean_error(function, path, excinfo):
+ if issubclass(OSError, excinfo[0]):
+ if excinfo[1].errno == errno.ENOENT:
+ # Who cares if the file we're deleting isn't there?
+ return
+ raise excinfo[1]
+
+ def need_to_run_gyp(platform):
+ try:
+ build_mtime = os.stat(platform.build_ninja()).st_mtime
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ return True
+ else:
+ raise e
+ pattern = re.compile('.*\.gyp[i]$')
+ for dirname, _, files in os.walk(os.path.join(Processor.aos_path(), '..')):
+ for f in [f for f in files if pattern.match(f)]:
+ if (os.stat(os.path.join(dirname, f)).st_mtime > build_mtime):
+ return True
+ return False
+
+ for platform in platforms:
+ print('Building %s...' % platform, file=sys.stderr)
+ if args.action_name == 'clean':
+ shutil.rmtree(platform.outdir(), onerror=handle_clean_error)
+ else:
+ if need_to_run_gyp(platform):
+ print('Running gyp...', file=sys.stderr)
+ gyp = subprocess.Popen(
+ (tools_config['GYP'],
+ '--check',
+ '--depth=%s' % os.path.join(Processor.aos_path(), '..'),
+ '--no-circular-check',
+ '-f', 'ninja',
+ '-I%s' % os.path.join(Processor.aos_path(), 'build', 'aos.gypi'),
+ '-I/dev/stdin', '-Goutput_dir=output',
+ '-DOS=%s' % platform.os(),
+ '-DPLATFORM=%s' % platform.gyp_platform(),
+ '-DARCHITECTURE=%s' % platform.architecture,
+ '-DCOMPILER=%s' % platform.compiler,
+ '-DDEBUG=%s' % ('yes' if platform.debug else 'no'),
+ args.main_gyp),
+ stdin=subprocess.PIPE)
+ gyp.communicate(("""
+{
+ 'target_defaults': {
+ 'configurations': {
+ '%s': {}
+ }
+ }
+}""" % platform.outname()).encode())
+ if gyp.returncode:
+ print("Running gyp failed!", file=sys.stderr)
+ exit(1)
+ print('Done running gyp.', file=sys.stderr)
+ else:
+ print("Not running gyp.", file=sys.stderr)
+
+ try:
+ subprocess.check_call(
+ (tools_config['NINJA'],
+ '-C', platform.outdir()) + tuple(targets),
+ stdin=open('/dev/null', 'r'))
+ except subprocess.CalledProcessError as e:
+ if unknown_platform_error is not None:
+ print(unknown_platform_error, file=sys.stderr)
+ raise e
+
+ # TODO(brians): deploy and tests
+ print('Done building %s...' % platform, file=sys.stderr)
+
+if __name__ == '__main__':
+ main()
diff --git a/aos/build/build.sh b/aos/build/build.sh
index 6bea1e6..d775105 100755
--- a/aos/build/build.sh
+++ b/aos/build/build.sh
@@ -29,11 +29,6 @@
${AOS}/build/download_externals.sh amd64
. $(dirname $0)/tools_config
-# The exciting quoting is so that it ends up with -DWHATEVER='"'`a command`'"'.
-# The '"' at either end is so that it creates a string constant when expanded
-# in the C/C++ code.
-COMMONFLAGS='-DLOG_SOURCENAME='"'\"'"'`basename $in`'"'\"' "
-
if [[ "${ACTION}" != "clean" && ( ! -d ${OUTDIR} || -n \
"`find ${AOS}/.. -newer ${BUILD_NINJA} \( -name '*.gyp' -or -name '*.gypi' \)`" ) ]]; then
echo 'Running gyp...' 1>&2
@@ -55,11 +50,6 @@
-DOS=$(echo ${PLATFORM} | sed 's/-.*//g') -DPLATFORM=${PLATFORM} \
-DWIND_BASE=${WIND_BASE} -DDEBUG=${DEBUG} \
${GYP_MAIN}
- # Have to substitute "command = $compiler" so that it doesn't try to
- # substitute them in the linker commands, where it doesn't work.
- sed -i "s:command = \$cc:\\0 ${COMMONFLAGS}:g ; \
- s:command = \$cxx:\\0 ${COMMONFLAGS}:g" \
- ${BUILD_NINJA}
if [ ${PLATFORM} == crio ]; then
sed -i 's/nm -gD/nm/g' ${BUILD_NINJA}
fi
diff --git a/aos/build/download_externals.sh b/aos/build/download_externals.sh
index e6e74f1..bf8df38 100755
--- a/aos/build/download_externals.sh
+++ b/aos/build/download_externals.sh
@@ -19,12 +19,19 @@
# Some of them need to set LDFLAGS separately to work around stupid configure
# scripts, so we can't just set that here.
CONFIGURE_FLAGS="--host=arm-linux-gnueabihf CC=${CC} CXX=${CXX} CFLAGS=\"${CFLAGS}\" CXXFLAGS=\"${CXXFLAGS}\" OBJDUMP=${OBJDUMP}"
-else
+ IS_CRIO=0
+elif [ "$1" == "amd64" ]; then
COMPILED=${EXTERNALS}/../compiled-amd64
export CFLAGS="-march=atom -mfpmath=sse"
export CXXFLAGS="-march=atom -mfpmath=sse"
CONFIGURE_FLAGS="CFLAGS=\"${CFLAGS}\" CXXFLAGS=\"${CXXFLAGS}\""
+ IS_CRIO=0
+elif [ "$1" == "crio" ]; then
+ IS_CRIO=1
+else
+ echo "Unknown platform $1 to download externals for." 1>&2
+ exit 1
fi
TMPDIR=/tmp/$$-aos-tmpdir
@@ -49,6 +56,22 @@
[ -f ${EIGEN_DIR}.tar.bz2 ] || wget http://bitbucket.org/eigen/eigen/get/${EIGEN_VERSION}.tar.bz2 -O ${EIGEN_DIR}.tar.bz2
[ -d ${EIGEN_DIR} ] || ( mkdir ${EIGEN_DIR} && tar --strip-components=1 -C ${EIGEN_DIR} -xf ${EIGEN_DIR}.tar.bz2 )
+# get the LLVM Compiler-RT source
+COMPILER_RT_TAG=RELEASE_32/final
+COMPILER_RT_VERSION=`echo ${COMPILER_RT_TAG} | sed s:/:_:`
+COMPILER_RT_DIR=${EXTERNALS}/compiler-rt-${COMPILER_RT_VERSION}
+COMPILER_RT_URL=http://llvm.org/svn/llvm-project/compiler-rt/tags/${COMPILER_RT_TAG}
+[ -d ${COMPILER_RT_DIR} ] || svn checkout ${COMPILER_RT_URL} ${COMPILER_RT_DIR}
+
+# get gtest
+GTEST_VERSION=1.6.0
+GTEST_DIR=${EXTERNALS}/gtest-${GTEST_VERSION}-p1
+GTEST_ZIP=${EXTERNALS}/gtest-${GTEST_VERSION}.zip
+[ -f ${GTEST_ZIP} ] || wget http://googletest.googlecode.com/files/gtest-${GTEST_VERSION}.zip -O ${GTEST_ZIP}
+[ -d ${GTEST_DIR} ] || ( unzip ${GTEST_ZIP} -d ${TMPDIR} && mv ${TMPDIR}/gtest-${GTEST_VERSION} ${GTEST_DIR} && cd ${GTEST_DIR} && patch -p1 < ${AOS}/externals/gtest.patch )
+
+[[ ${IS_CRIO} -eq 1 ]] && exit 0
+
# get and build libjpeg
LIBJPEG_VERSION=8d
LIBJPEG_DIR=${COMPILED}/jpeg-${LIBJPEG_VERSION}
@@ -63,13 +86,6 @@
${CONFIGURE_FLAGS} --prefix=`readlink -f ${LIBJPEG_PREFIX}` \
&& make && make install"
-# get gtest
-GTEST_VERSION=1.6.0
-GTEST_DIR=${EXTERNALS}/gtest-${GTEST_VERSION}-p1
-GTEST_ZIP=${EXTERNALS}/gtest-${GTEST_VERSION}.zip
-[ -f ${GTEST_ZIP} ] || wget http://googletest.googlecode.com/files/gtest-${GTEST_VERSION}.zip -O ${GTEST_ZIP}
-[ -d ${GTEST_DIR} ] || ( unzip ${GTEST_ZIP} -d ${TMPDIR} && mv ${TMPDIR}/gtest-${GTEST_VERSION} ${GTEST_DIR} && cd ${GTEST_DIR} && patch -p1 < ${AOS}/externals/gtest.patch )
-
# get and build ctemplate
# This is the next revision after the 2.2 release and it only adds spaces to
# make gcc 4.7 with --std=c++11 happy (user-defined string literals...).
@@ -109,13 +125,6 @@
${CONFIGURE_FLAGS} --prefix=`readlink -f ${GFLAGS_PREFIX}` \
&& make && make install"
-# get the LLVM Compiler-RT source
-COMPILER_RT_TAG=RELEASE_32/final
-COMPILER_RT_VERSION=`echo ${COMPILER_RT_TAG} | sed s:/:_:`
-COMPILER_RT_DIR=${EXTERNALS}/compiler-rt-${COMPILER_RT_VERSION}
-COMPILER_RT_URL=http://llvm.org/svn/llvm-project/compiler-rt/tags/${COMPILER_RT_TAG}
-[ -d ${COMPILER_RT_DIR} ] || svn checkout ${COMPILER_RT_URL} ${COMPILER_RT_DIR}
-
# get and build libevent
LIBEVENT_VERSION=2.0.21
LIBEVENT_TAR=${EXTERNALS}/libevent-${LIBEVENT_VERSION}.tar.gz
diff --git a/aos/build/externals.gyp b/aos/build/externals.gyp
index 536cc19..8e4e637 100644
--- a/aos/build/externals.gyp
+++ b/aos/build/externals.gyp
@@ -5,8 +5,8 @@
# TODO(brians): Would we not have to do this hackery if we named it externals_path etc?
'externals': '<(AOS)/../output/downloaded',
'externals_abs': '<!(readlink -f ../../output/downloaded)',
- 'compiled': '<(externals)/../compiled-<(aos_architecture)',
- 'compiled_abs': '<(externals_abs)/../compiled-<(aos_architecture)',
+ 'compiled': '<(externals)/../compiled-<(ARCHITECTURE)',
+ 'compiled_abs': '<(externals_abs)/../compiled-<(ARCHITECTURE)',
# These versions have to be kept in sync with the ones in download_externals.sh.
'eigen_version': '3.2.1',
diff --git a/aos/build/prime-build.sh b/aos/build/prime-build.sh
deleted file mode 100755
index f1aea44..0000000
--- a/aos/build/prime-build.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-#set -x
-# TODO(brians): This needs to be a python script that still deals with
-# stdout/stderr correctly but also has real argument parsing.
-# That should probably fold most of the other shell scripts in too...
-
-if [[ "$1" == "amd64" ]]; then
- NO_TARGET=1
- shift 1
-elif [[ "$1" == "arm" ]]; then
- NO_AMD64=1
- shift 1
-fi
-
-[[ "$1" == "tests" ]] && NO_TARGET=1
-[[ "$1" == "deploy" ]] && NO_AMD64=1
-
-OTHER_ARGS="$@"
-
-build_platform() {
- PLATFORM=$1
- OUT_NAME=$2
- HUMAN_NAME=$3
-
- echo "Building code with clang for ${HUMAN_NAME}..." 1>&2
- ../../aos/build/build.sh ${PLATFORM}-clang prime.gyp no ${OUT_NAME}-clang "${OTHER_ARGS}"
- if [[ $? -ne 0 ]]; then
- echo "Building code with clang for ${HUMAN_NAME} failed!" 1>&2
- exit 1
- fi
- echo "Building code with clang for ${HUMAN_NAME} succeeded." 1>&2
-
- echo "Building code for ${HUMAN_NAME}..." 1>&2
- ../../aos/build/build.sh ${PLATFORM} prime.gyp no ${OUT_NAME} "${OTHER_ARGS}"
- if [[ $? -ne 0 ]]; then
- echo "Building code for ${HUMAN_NAME} failed!" 1>&2
- exit 1
- fi
- echo "Building code for ${HUMAN_NAME} succeeded." 1>&2
-}
-
-if [[ ! ${NO_TARGET} ]]; then
- build_platform linux prime target
-fi
-if [[ ! ${NO_AMD64} ]]; then
- build_platform linux-amd64 prime-amd64 amd64
-fi
diff --git a/aos/build/tools_config b/aos/build/tools_config
index d47a3ce..88931af 100644
--- a/aos/build/tools_config
+++ b/aos/build/tools_config
@@ -1,5 +1,7 @@
# This is a shell fragment that sets up variables related to where the tools
# that download_externals.sh downloads so build.sh can use.
+# It also gets parsed by build.py, so it has to stay just variable assignments
+# and comments.
EXTERNALS=${AOS}/../output/downloaded
diff --git a/aos/common/logging/logging.h b/aos/common/logging/logging.h
index 6487b64..03f3e31 100644
--- a/aos/common/logging/logging.h
+++ b/aos/common/logging/logging.h
@@ -59,7 +59,6 @@
//#define LOG_CURRENT_FUNCTION __PRETTY_FUNCTION__
#define LOG_CURRENT_FUNCTION __func__
-#undef LOG_SOURCENAME
#define LOG_SOURCENAME __FILE__
// The basic logging call.
diff --git a/frc971/prime/build.sh b/frc971/prime/build.sh
index 71b6e32..0a1b0ba 100755
--- a/frc971/prime/build.sh
+++ b/frc971/prime/build.sh
@@ -2,4 +2,4 @@
cd $(dirname $0)
-../../aos/build/prime-build.sh "$@"
+../../aos/build/build.py --processor prime --main_gyp prime.gyp "$@"
diff --git a/output/.gitignore b/output/.gitignore
index 79c1adf..f0ed9a5 100644
--- a/output/.gitignore
+++ b/output/.gitignore
@@ -1,6 +1,9 @@
/prime/
/crio/
-/prime-amd64/
/flasher/
/compiled-*/
/ip_base.txt
+/amd64-clang/
+/arm-clang/
+/amd64-gcc/
+/arm-gcc/