Merge "add support for replaying log messages"
diff --git a/aos/build/aos.gypi b/aos/build/aos.gypi
index 22e96a0..e049182 100644
--- a/aos/build/aos.gypi
+++ b/aos/build/aos.gypi
@@ -24,13 +24,13 @@
],
},
'conditions': [
- ['PLATFORM=="linux-arm-gcc_frc"', {
+ ['PLATFORM=="linux-arm_frc-gcc"', {
'make_global_settings': [
['CC', '<(ccache)<!(which arm-frc-linux-gnueabi-gcc-4.9)'],
['CXX', '<(ccache)<!(which arm-frc-linux-gnueabi-g++-4.9)'],
],
},
- ], ['PLATFORM=="linux-arm-clang_frc"', {
+ ], ['PLATFORM=="linux-arm_frc-clang"', {
'variables': {
'arm-clang-symlinks': '<!(realpath -s <(AOS)/build/arm-clang-symlinks)',
'arm-clang-sysroot': '<(arm-clang-symlinks)/sysroot',
@@ -140,6 +140,7 @@
'__STDC_CONSTANT_MACROS',
'__STDC_LIMIT_MACROS',
'AOS_COMPILER_<!(echo <(FULL_COMPILER) | sed \'s/\./_/g\')',
+ 'AOS_ARCHITECTURE_<(ARCHITECTURE)',
'_FILE_OFFSET_BITS=64',
],
'ldflags': [
diff --git a/aos/build/build.py b/aos/build/build.py
index 21e0379..6281821 100755
--- a/aos/build/build.py
+++ b/aos/build/build.py
@@ -137,56 +137,24 @@
"""
return os.path.join(os.path.dirname(__file__), '..')
-def get_ip_base():
- """Retrieves the IP address base."""
+def get_ip():
+ """Retrieves the IP address to download code to."""
FILENAME = os.path.normpath(os.path.join(aos_path(), '..',
- 'output', 'ip_base.txt'))
+ 'output', 'ip_address.txt'))
if not os.access(FILENAME, os.R_OK):
os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
with open(FILENAME, 'w') as f:
f.write('roboRIO-971.local')
with open(FILENAME, 'r') as f:
- base = f.readline().strip()
- return base
+ return f.readline().strip()
-def get_ip(device):
- """Retrieves the IP address for a given device."""
- base = get_ip_base()
- if device == 'prime':
- return base + '.179'
- elif device == 'robot':
- return base + '.2'
- elif device == 'roboRIO':
- return base
- else:
- raise Exception('Unknown device %s to get an IP address for.' % device)
+def get_temp_dir():
+ """Retrieves the temporary directory to use when downloading."""
+ return '/home/admin/tmp/aos_downloader'
-def get_user(device):
- """Retrieves the user for a given device."""
- if device == 'prime':
- return 'driver'
- elif device == 'roboRIO':
- return 'admin'
- else:
- raise Exception('Unknown device %s to get a user for.' % device)
-
-def get_temp_dir(device):
- """Retrieves the temporary download directory for a given device."""
- if device == 'prime':
- return '/tmp/aos_downloader'
- elif device == 'roboRIO':
- return '/home/admin/tmp/aos_downloader'
- else:
- raise Exception('Unknown device %s to get a temp_dir for.' % device)
-
-def get_target_dir(device):
- """Retrieves the tempory deploy directory for a given device."""
- if device == 'prime':
- return '/home/driver/robot_code/bin'
- elif device == 'roboRIO':
- return '/home/admin/robot_code'
- else:
- raise Exception('Unknown device %s to get a temp_dir for.' % device)
+def get_target_dir():
+ """Retrieves the tempory deploy directory for downloading code."""
+ return '/home/admin/robot_code'
def user_output(message):
"""Prints message to the user."""
@@ -434,16 +402,15 @@
return r
def deploy(self, dry_run):
- # Downloads code to the prime in a way that avoids clashing too badly with
- # starter (like the naive download everything one at a time).
- if self.compiler().endswith('_frc'):
- device = 'roboRIO'
- else:
- device = 'prime'
+ """Downloads code to the prime in a way that avoids clashing too badly with
+ starter (like the naive download everything one at a time)."""
+ if not self.architecture().endswith('_frc'):
+ raise Exception("Don't know how to download code to a %s." %
+ self.architecture())
SUM = 'md5sum'
- TARGET_DIR = get_target_dir(device)
- TEMP_DIR = get_temp_dir(device)
- TARGET = get_user(device) + '@' + get_ip(device)
+ TARGET_DIR = get_target_dir()
+ TEMP_DIR = get_temp_dir()
+ TARGET = 'admin@' + get_ip()
from_dir = os.path.join(self.outdir(), 'outputs')
sums = subprocess.check_output((SUM,) + tuple(os.listdir(from_dir)),
@@ -467,8 +434,7 @@
+ (('%s:%s' % (TARGET, TEMP_DIR)),))
if not dry_run:
mv_cmd = ['mv {TMPDIR}/* {TO_DIR} ']
- if device == 'roboRIO':
- mv_cmd.append('&& chmod u+s {TO_DIR}/starter_exe ')
+ mv_cmd.append('&& chmod u+s {TO_DIR}/starter_exe ')
mv_cmd.append('&& echo \'Done moving new executables into place\' ')
mv_cmd.append('&& bash -c \'sync && sync && sync\'')
subprocess.check_call(
@@ -511,8 +477,8 @@
return r
- ARCHITECTURES = ('arm', 'amd64')
- COMPILERS = ('clang', 'gcc_frc', 'clang_frc')
+ ARCHITECTURES = ('arm_frc', 'amd64')
+ COMPILERS = ('clang', 'gcc')
SANITIZERS = ('address', 'undefined', 'integer', 'memory', 'thread', 'none')
SANITIZER_TEST_WARNINGS = {
'memory': (True,
@@ -529,8 +495,7 @@
for architecture in PrimeProcessor.ARCHITECTURES:
for compiler in PrimeProcessor.COMPILERS:
for debug in [True, False]:
- if ((architecture == 'arm' and not compiler.endswith('_frc')) or
- (architecture == 'amd64' and compiler.endswith('_frc'))):
+ if architecture == 'amd64' and compiler == 'gcc':
# We don't have a compiler to use here.
continue
platforms.append(
@@ -556,8 +521,8 @@
if warning[0]:
default_platforms -= self.select_platforms(sanitizer=sanitizer)
elif is_deploy:
- default_platforms = self.select_platforms(architecture='arm',
- compiler='gcc_frc',
+ default_platforms = self.select_platforms(architecture='arm_frc',
+ compiler='gcc',
debug=False)
else:
default_platforms = self.select_platforms(debug=False)
@@ -578,13 +543,8 @@
if platforms & pie_sanitizers:
to_download.add(architecture + '-fPIE')
- frc_platforms = self.select_platforms(architecture=architecture,
- compiler='gcc_frc')
- if platforms & frc_platforms:
- to_download.add(architecture + '_frc')
-
if platforms & (self.select_platforms(architecture=architecture) -
- pie_sanitizers - frc_platforms):
+ pie_sanitizers):
to_download.add(architecture)
for download_target in to_download:
@@ -651,14 +611,13 @@
packages.add('clang-3.5')
packages.add('clang-format-3.5')
for platform in platforms:
- if (platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8' or
- platform.compiler() == 'clang_frc'):
+ if platform.compiler() == 'clang' or platform.compiler() == 'gcc_4.8':
packages.add('clang-3.5')
if platform.compiler() == 'gcc_4.8':
packages.add('libcloog-isl3:amd64')
if is_deploy:
packages.add('openssh-client')
- elif platform.compiler() == 'gcc_frc' or platform.compiler() == 'clang_frc':
+ elif platform.architecture == 'arm_frc':
packages.add('gcc-4.9-arm-frc-linux-gnueabi')
packages.add('g++-4.9-arm-frc-linux-gnueabi')
@@ -933,7 +892,7 @@
'-DSANITIZER=%s' % platform.sanitizer(),
'-DEXTERNALS_EXTRA=%s' %
('-fPIE' if platform.sanitizer() in PrimeProcessor.PIE_SANITIZERS
- else ('_frc' if platform.compiler().endswith('_frc') else ''))) +
+ else '')) +
processor.extra_gyp_flags() + (args.main_gyp,),
stdin=subprocess.PIPE)
gyp.communicate(("""
diff --git a/aos/common/controls/control_loop-tmpl.h b/aos/common/controls/control_loop-tmpl.h
index 44fada1..449ffea 100644
--- a/aos/common/controls/control_loop-tmpl.h
+++ b/aos/common/controls/control_loop-tmpl.h
@@ -38,11 +38,7 @@
// Fetch the latest control loop goal. If there is no new
// goal, we will just reuse the old one.
- // If there is no goal, we haven't started up fully. It isn't worth
- // the added complexity for each loop implementation to handle that case.
control_loop_->goal.FetchLatest();
- // TODO(aschuh): Check the age here if we want the loop to stop on old
- // goals.
const GoalType *goal = control_loop_->goal.get();
if (goal) {
LOG_STRUCT(DEBUG, "goal", *goal);
diff --git a/aos/linux_code/logging/binary_log_file.cc b/aos/linux_code/logging/binary_log_file.cc
index 1f9815e..b063fae 100644
--- a/aos/linux_code/logging/binary_log_file.cc
+++ b/aos/linux_code/logging/binary_log_file.cc
@@ -35,6 +35,30 @@
msync(current_, kPageSize, MS_ASYNC | MS_INVALIDATE);
}
+void LogFileAccessor::SkipToLastSeekablePage() {
+ CHECK(definitely_use_mmap());
+
+ struct stat info;
+ if (fstat(fd_, &info) == -1) {
+ PLOG(FATAL, "fstat(%d, %p) failed", fd_, &info);
+ }
+
+ CHECK((info.st_size % kPageSize) == 0);
+ const auto last_readable_page_number = (info.st_size / kPageSize) - 1;
+ const auto last_seekable_page_number =
+ last_readable_page_number / kSeekPages * kSeekPages;
+ const off_t new_offset = last_seekable_page_number * kPageSize;
+ // We don't want to go backwards...
+ if (new_offset > offset_) {
+ Unmap(current_);
+ offset_ = new_offset;
+ MapNextPage();
+ }
+}
+
+// The only way to tell is using fstat, but we don't really want to be making a
+// syscall every single time somebody wants to know the answer, so it gets
+// cached in is_last_page_.
bool LogFileAccessor::IsLastPage() {
if (is_last_page_ != Maybe::kUnknown) {
return is_last_page_ == Maybe::kYes;
@@ -116,6 +140,7 @@
r = static_cast<LogFileMessageHeader *>(
static_cast<void *>(¤t()[position()]));
if (wait) {
+ CHECK(definitely_use_mmap());
if (futex_wait(&r->marker) != 0) continue;
}
if (r->marker == 2) {
@@ -194,23 +219,45 @@
}
LogFileMessageHeader *LogFileWriter::GetWritePosition(size_t message_size) {
- if (position() + message_size + (kAlignment - (message_size % kAlignment)) +
- sizeof(aos_futex) > kPageSize) {
- char *const temp = current();
- MapNextPage();
- if (futex_set_value(static_cast<aos_futex *>(static_cast<void *>(
- &temp[position()])), 2) == -1) {
- PLOG(WARNING, "readers will hang because futex_set_value(%p, 2) failed",
- &temp[position()]);
- }
- Unmap(temp);
- }
+ if (NeedNewPageFor(message_size)) ForceNewPage();
LogFileMessageHeader *const r = static_cast<LogFileMessageHeader *>(
static_cast<void *>(¤t()[position()]));
IncrementPosition(message_size);
return r;
}
+// A number of seekable pages, not the actual file offset, is stored in *cookie.
+bool LogFileWriter::ShouldClearSeekableData(off_t *cookie,
+ size_t next_message_size) const {
+ off_t next_message_page = (offset() / kPageSize) - 1;
+ if (NeedNewPageFor(next_message_size)) {
+ ++next_message_page;
+ }
+ const off_t current_seekable_page = next_message_page / kSeekPages;
+ CHECK_LE(*cookie, current_seekable_page);
+ const bool r = *cookie != current_seekable_page;
+ *cookie = current_seekable_page;
+ return r;
+}
+
+bool LogFileWriter::NeedNewPageFor(size_t bytes) const {
+ return position() + bytes + (kAlignment - (bytes % kAlignment)) +
+ sizeof(aos_futex) >
+ kPageSize;
+}
+
+void LogFileWriter::ForceNewPage() {
+ char *const temp = current();
+ MapNextPage();
+ if (futex_set_value(
+ static_cast<aos_futex *>(static_cast<void *>(&temp[position()])),
+ 2) == -1) {
+ PLOG(WARNING, "readers will hang because futex_set_value(%p, 2) failed",
+ &temp[position()]);
+ }
+ Unmap(temp);
+}
+
} // namespace linux_code
} // namespace logging
} // namespace aos
diff --git a/aos/linux_code/logging/binary_log_file.h b/aos/linux_code/logging/binary_log_file.h
index 84f8398..bcf9da1 100644
--- a/aos/linux_code/logging/binary_log_file.h
+++ b/aos/linux_code/logging/binary_log_file.h
@@ -94,8 +94,17 @@
// Asynchronously syncs all open mappings.
void Sync() const;
+ // Returns true iff we currently have the last page in the file mapped.
+ // This is fundamentally a racy question, so the return value may not be
+ // accurate by the time this method returns.
bool IsLastPage();
+ // Skips to the last page which is an even multiple of kSeekPages.
+ // This is fundamentally racy, so it may not actually be on the very last
+ // possible multiple of kSeekPages when it returns, but it should be close.
+ // This will never move backwards.
+ void SkipToLastSeekablePage();
+
size_t file_offset(const void *msg) {
return offset() + (static_cast<const char *>(msg) - current());
}
@@ -109,6 +118,10 @@
// What to align messages to, copied into an actual constant.
static const size_t kAlignment = MESSAGE_ALIGNMENT;
#undef MESSAGE_ALIGNMENT
+ // Pages which are multiples of this from the beginning of a file start with
+ // no saved state (ie struct types). This allows seeking immediately to the
+ // largest currently written interval of this number when following.
+ static const size_t kSeekPages = 256;
char *current() const { return current_; }
size_t position() const { return position_; }
@@ -169,6 +182,22 @@
// message_size should be the total number of bytes needed for the message.
LogFileMessageHeader *GetWritePosition(size_t message_size);
+
+ // Returns true exactly once for each unique cookie on each page where cached
+ // data should be cleared.
+ // Call with a non-zero next_message_size to determine if cached data should
+ // be forgotten before writing a next_message_size-sized message.
+ // cookie should be initialized to 0.
+ bool ShouldClearSeekableData(off_t *cookie, size_t next_message_size) const;
+
+ // Forces a move to a new page for the next message.
+ // This is important when there is cacheable data that needs to be re-written
+ // before a message which will spill over onto the next page but the cacheable
+ // message being refreshed is smaller and won't get to a new page by itself.
+ void ForceNewPage();
+
+ private:
+ bool NeedNewPageFor(size_t bytes) const;
};
} // namespace linux_code
diff --git a/aos/linux_code/logging/binary_log_writer.cc b/aos/linux_code/logging/binary_log_writer.cc
index cc3cf66..dbe61e0 100644
--- a/aos/linux_code/logging/binary_log_writer.cc
+++ b/aos/linux_code/logging/binary_log_writer.cc
@@ -26,14 +26,14 @@
namespace linux_code {
namespace {
-void CheckTypeWritten(uint32_t type_id, LogFileWriter &writer) {
- static ::std::unordered_set<uint32_t> written_type_ids;
- if (written_type_ids.count(type_id) > 0) return;
+void CheckTypeWritten(uint32_t type_id, LogFileWriter *writer,
+ ::std::unordered_set<uint32_t> *written_type_ids) {
+ if (written_type_ids->count(type_id) > 0) return;
if (MessageType::IsPrimitive(type_id)) return;
const MessageType &type = type_cache::Get(type_id);
for (int i = 0; i < type.number_fields; ++i) {
- CheckTypeWritten(type.fields[i]->type, writer);
+ CheckTypeWritten(type.fields[i]->type, writer, written_type_ids);
}
char buffer[1024];
@@ -44,7 +44,7 @@
return;
}
LogFileMessageHeader *const output =
- writer.GetWritePosition(sizeof(LogFileMessageHeader) + size);
+ writer->GetWritePosition(sizeof(LogFileMessageHeader) + size);
output->time_sec = output->time_nsec = 0;
output->source = getpid();
@@ -58,7 +58,7 @@
output->type = LogFileMessageHeader::MessageType::kStructType;
futex_set(&output->marker);
- written_type_ids.insert(type_id);
+ written_type_ids->insert(type_id);
}
void AllocateLogName(char **filename, const char *directory) {
@@ -108,7 +108,7 @@
fileindex, directory, previous);
}
-#ifdef AOS_COMPILER_gcc_frc
+#ifdef AOS_ARCHITECTURE_arm_frc
bool FoundThumbDrive(const char *path) {
FILE *mnt_fp = setmntent("/etc/mtab", "r");
if (mnt_fp == nullptr) {
@@ -148,7 +148,7 @@
int BinaryLogReaderMain() {
InitNRT();
-#ifdef AOS_COMPILER_gcc_frc
+#ifdef AOS_ARCHITECTURE_arm_frc
char folder[128];
{
@@ -198,6 +198,9 @@
}
LogFileWriter writer(fd);
+ ::std::unordered_set<uint32_t> written_type_ids;
+ off_t clear_type_ids_cookie = 0;
+
while (true) {
const LogMessage *const msg = ReadNext();
if (msg == NULL) continue;
@@ -208,7 +211,12 @@
if (msg->type == LogMessage::Type::kStruct) {
output_length += sizeof(msg->structure.type_id) + sizeof(uint32_t) +
msg->structure.string_length;
- CheckTypeWritten(msg->structure.type_id, writer);
+ if (writer.ShouldClearSeekableData(&clear_type_ids_cookie,
+ output_length)) {
+ writer.ForceNewPage();
+ written_type_ids.clear();
+ }
+ CheckTypeWritten(msg->structure.type_id, &writer, &written_type_ids);
} else if (msg->type == LogMessage::Type::kMatrix) {
output_length +=
sizeof(msg->matrix.type) + sizeof(uint32_t) + sizeof(uint16_t) +
diff --git a/aos/linux_code/logging/log_displayer.cc b/aos/linux_code/logging/log_displayer.cc
index 22ea0bd..cdc8f46 100644
--- a/aos/linux_code/logging/log_displayer.cc
+++ b/aos/linux_code/logging/log_displayer.cc
@@ -233,16 +233,20 @@
}
}
- fprintf(stderr, "displaying down to level %s from file '%s'\n",
- ::aos::logging::log_str(filter_level), filename);
-
int fd;
if (strcmp(filename, "-") == 0) {
+ if (skip_to_end) {
+ fputs("Can't skip to end of stdin!\n", stderr);
+ return EXIT_FAILURE;
+ }
fd = STDIN_FILENO;
} else {
fd = open(filename, O_RDONLY);
}
+ fprintf(stderr, "displaying down to level %s from file '%s'\n",
+ ::aos::logging::log_str(filter_level), filename);
+
if (fd == -1) {
PLOG(FATAL, "couldn't open file '%s' for reading", filename);
}
@@ -250,6 +254,7 @@
if (skip_to_end) {
fputs("skipping old logs...\n", stderr);
+ reader.SkipToLastSeekablePage();
}
const LogFileMessageHeader *msg;
diff --git a/frc971/analysis/analysis.py b/frc971/analysis/analysis.py
index 66c5056..8152607 100755
--- a/frc971/analysis/analysis.py
+++ b/frc971/analysis/analysis.py
@@ -70,7 +70,8 @@
"""
for key in self.signal:
value = self.signal[key]
- pylab.plot(value.time, value.data, label=key[0] + ' ' + '.'.join(key[2]))
+ pylab.plot(value.time, value.data, label=key[0] + ' ' + key[1] + '.' + \
+ '.'.join(key[2]))
pylab.legend()
pylab.show()
diff --git a/output/.gitignore b/output/.gitignore
index b617a67..d675b14 100644
--- a/output/.gitignore
+++ b/output/.gitignore
@@ -2,14 +2,14 @@
/crio/
/flasher/
/compiled-*/
-/ip_base.txt
+/ip_address.txt
/ccache_dir/
/crio/
/crio-debug/
-/arm-[^-]*-[^-]*/
-/arm-[^-]*-debug-[^-]*/
+/arm_frc-[^-]*-[^-]*/
+/arm_frc-[^-]*-debug-[^-]*/
/amd64-[^-]*-[^-]*/
/amd64-[^-]*-debug-[^-]*/
/bot3-[^-]*-[^-]*-[^-]*/