Removed Common
Change-Id: I01ea8f07220375c2ad9bc0092281d4f27c642303
diff --git a/aos/logging/BUILD b/aos/logging/BUILD
new file mode 100644
index 0000000..a8ee8e9
--- /dev/null
+++ b/aos/logging/BUILD
@@ -0,0 +1,209 @@
+# The primary client logging interface.
+cc_library(
+ name = "logging",
+ srcs = [
+ "context.cc",
+ "interface.cc",
+ ],
+ hdrs = [
+ "context.h",
+ "interface.h",
+ "logging.h",
+ ],
+ compatible_with = [
+ "//tools:armhf-debian",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":sizes",
+ "//aos:die",
+ "//aos:macros",
+ "//aos/libc:aos_strerror",
+ "//aos/linux_code:complex_thread_local",
+ ],
+)
+
+cc_library(
+ name = "replay",
+ srcs = [
+ "replay.cc",
+ ],
+ hdrs = [
+ "replay.h",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":binary_log_file",
+ ":logging",
+ "//aos:queues",
+ "//aos/linux_code/ipc_lib:queue",
+ ],
+)
+
+cc_binary(
+ name = "binary_log_writer",
+ srcs = [
+ "binary_log_writer.cc",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":binary_log_file",
+ ":implementations",
+ ":logging",
+ "//aos:die",
+ "//aos:queue_types",
+ "//aos/time:time",
+ "//aos/linux_code:configuration",
+ "//aos/linux_code:init",
+ "//aos/linux_code/ipc_lib:queue",
+ ],
+)
+
+cc_binary(
+ name = "log_streamer",
+ srcs = [
+ "log_streamer.cc",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":implementations",
+ ":logging",
+ "//aos/time:time",
+ "//aos/linux_code:init",
+ "//aos/linux_code/ipc_lib:queue",
+ ],
+)
+
+cc_binary(
+ name = "log_displayer",
+ srcs = [
+ "log_displayer.cc",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":binary_log_file",
+ ":implementations",
+ ":logging",
+ "//aos:queue_types",
+ "//aos/util:string_to_num",
+ "//aos/linux_code:configuration",
+ "//aos/linux_code:init",
+ ],
+)
+
+cc_library(
+ name = "binary_log_file",
+ srcs = [
+ "binary_log_file.cc",
+ ],
+ hdrs = [
+ "binary_log_file.h",
+ ],
+ deps = [
+ ":implementations",
+ ],
+)
+
+cc_library(
+ name = "sizes",
+ hdrs = [
+ "sizes.h",
+ ],
+ compatible_with = [
+ "//tools:armhf-debian",
+ ],
+)
+
+cc_test(
+ name = "implementations_test",
+ srcs = [
+ "implementations_test.cc",
+ ],
+ deps = [
+ ":implementations",
+ ":logging",
+ "//aos/testing:googletest",
+ ],
+)
+
+cc_library(
+ name = "queue_logging",
+ srcs = [
+ "queue_logging.cc",
+ ],
+ hdrs = [
+ "queue_logging.h",
+ ],
+ compatible_with = [
+ "//tools:armhf-debian",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":logging",
+ ":sizes",
+ "//aos:die",
+ "//aos:queue_types",
+ ],
+)
+
+cc_library(
+ name = "matrix_logging",
+ srcs = [
+ "matrix_logging.cc",
+ ],
+ hdrs = [
+ "matrix_logging.h",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":logging",
+ ":sizes",
+ "//aos:die",
+ "//aos:generated_queue_headers",
+ "//aos:queue_types",
+ "//third_party/eigen",
+ ],
+)
+
+cc_library(
+ name = "printf_formats",
+ hdrs = [
+ "printf_formats.h",
+ ],
+ compatible_with = [
+ "//tools:armhf-debian",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ "//aos:macros",
+ ],
+)
+
+cc_library(
+ name = "implementations",
+ srcs = [
+ "implementations.cc",
+ ],
+ hdrs = [
+ "implementations.h",
+ ],
+ compatible_with = [
+ "//tools:armhf-debian",
+ ],
+ linkopts = [
+ "-lpthread",
+ ],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":logging",
+ ":sizes",
+ "//aos:once",
+ "//aos:die",
+ "//aos:macros",
+ "//aos/mutex:mutex",
+ "//aos:queue_types",
+ "//aos/time:time",
+ "//aos/type_traits:type_traits",
+ "//aos/linux_code/ipc_lib:queue",
+ ],
+)
diff --git a/aos/logging/binary_log_file.cc b/aos/logging/binary_log_file.cc
new file mode 100644
index 0000000..ca62b73
--- /dev/null
+++ b/aos/logging/binary_log_file.cc
@@ -0,0 +1,263 @@
+#include "aos/logging/binary_log_file.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <signal.h>
+#include <setjmp.h>
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+namespace {
+
+unsigned long SystemPageSize() {
+ static unsigned long r = sysconf(_SC_PAGESIZE);
+ return r;
+}
+
+} // namespace
+
+LogFileAccessor::LogFileAccessor(int fd, bool writable)
+ : fd_(fd), writable_(writable), offset_(0), current_(0), position_(0) {
+ // Check to make sure that mmap will allow mmaping in chunks of kPageSize.
+ if (SystemPageSize() > kPageSize || (kPageSize % SystemPageSize()) != 0) {
+ LOG(FATAL, "system page size (%lu) not factor of kPageSize (%zd).\n",
+ SystemPageSize(), kPageSize);
+ }
+
+ MapNextPage();
+}
+
+void LogFileAccessor::Sync() const {
+ msync(current_, kPageSize, MS_ASYNC | MS_INVALIDATE);
+}
+
+void LogFileAccessor::SkipToLastSeekablePage() {
+ CHECK(definitely_use_mmap());
+
+ struct stat info;
+ if (fstat(fd_, &info) == -1) {
+ PLOG(FATAL, "fstat(%d, %p) failed", fd_, &info);
+ }
+
+ CHECK((info.st_size % kPageSize) == 0);
+ const auto last_readable_page_number = (info.st_size / kPageSize) - 1;
+ const auto last_seekable_page_number =
+ last_readable_page_number / kSeekPages * kSeekPages;
+ const off_t new_offset = last_seekable_page_number * kPageSize;
+ // We don't want to go backwards...
+ if (new_offset > offset_) {
+ Unmap(current_);
+ offset_ = new_offset;
+ MapNextPage();
+ }
+}
+
+// The only way to tell is using fstat, but we don't really want to be making a
+// syscall every single time somebody wants to know the answer, so it gets
+// cached in is_last_page_.
+bool LogFileAccessor::IsLastPage() {
+ if (is_last_page_ != Maybe::kUnknown) {
+ return is_last_page_ == Maybe::kYes;
+ }
+
+ struct stat info;
+ if (fstat(fd_, &info) == -1) {
+ PLOG(FATAL, "fstat(%d, %p) failed", fd_, &info);
+ }
+ bool r = offset_ == static_cast<off_t>(info.st_size - kPageSize);
+ is_last_page_ = r ? Maybe::kYes : Maybe::kNo;
+ return r;
+}
+
+void LogFileAccessor::MapNextPage() {
+ if (writable_) {
+ if (ftruncate(fd_, offset_ + kPageSize) == -1) {
+ PLOG(FATAL, "ftruncate(%d, %zd) failed", fd_, kPageSize);
+ }
+ }
+
+ if (use_read_ == Maybe::kYes) {
+ ssize_t todo = kPageSize;
+ while (todo > 0) {
+ ssize_t result = read(fd_, current_ + (kPageSize - todo), todo);
+ if (result == -1) {
+ PLOG(FATAL, "read(%d, %p, %zu) failed", fd_,
+ current_ + (kPageSize - todo), todo);
+ } else if (result == 0) {
+ memset(current_, 0, todo);
+ result = todo;
+ }
+ todo -= result;
+ }
+ CHECK_EQ(0, todo);
+ } else {
+ current_ = static_cast<char *>(
+ mmap(NULL, kPageSize, PROT_READ | (writable_ ? PROT_WRITE : 0),
+ MAP_SHARED, fd_, offset_));
+ if (current_ == MAP_FAILED) {
+ if (!writable_ && use_read_ == Maybe::kUnknown && errno == ENODEV) {
+ LOG(INFO, "Falling back to reading the file using read(2).\n");
+ use_read_ = Maybe::kYes;
+ current_ = new char[kPageSize];
+ MapNextPage();
+ return;
+ } else {
+ PLOG(FATAL,
+ "mmap(NULL, %zd, PROT_READ [ | PROT_WRITE], MAP_SHARED, %d, %jd)"
+ " failed",
+ kPageSize, fd_, static_cast<intmax_t>(offset_));
+ }
+ } else {
+ use_read_ = Maybe::kNo;
+ }
+ if (madvise(current_, kPageSize, MADV_SEQUENTIAL | MADV_WILLNEED) == -1) {
+ PLOG(WARNING, "madvise(%p, %zd, MADV_SEQUENTIAL | MADV_WILLNEED) failed",
+ current_, kPageSize);
+ }
+ }
+ offset_ += kPageSize;
+}
+
+void LogFileAccessor::Unmap(void *location) {
+ CHECK_NE(Maybe::kUnknown, use_read_);
+
+ if (use_read_ == Maybe::kNo) {
+ if (munmap(location, kPageSize) == -1) {
+ PLOG(FATAL, "munmap(%p, %zd) failed", location, kPageSize);
+ }
+ }
+ is_last_page_ = Maybe::kUnknown;
+ position_ = 0;
+}
+
+const LogFileMessageHeader *LogFileReader::ReadNextMessage(bool wait) {
+ LogFileMessageHeader *r;
+ do {
+ r = static_cast<LogFileMessageHeader *>(
+ static_cast<void *>(¤t()[position()]));
+ if (wait) {
+ CHECK(definitely_use_mmap());
+ if (futex_wait(&r->marker) != 0) continue;
+ }
+ if (r->marker == 2) {
+ Unmap(current());
+ MapNextPage();
+ CheckCurrentPageReadable();
+ r = static_cast<LogFileMessageHeader *>(static_cast<void *>(current()));
+ }
+ } while (wait && r->marker == 0);
+ if (r->marker == 0) {
+ return NULL;
+ }
+ IncrementPosition(sizeof(LogFileMessageHeader) + r->name_size +
+ r->message_size);
+ if (position() >= kPageSize) {
+ // It's a lot better to blow up here rather than getting SIGBUS errors the
+ // next time we try to read...
+ LOG(FATAL, "corrupt log file running over page size\n");
+ }
+ return r;
+}
+
+// This mess is because the only not completely hackish way to do this is to set
+// up a handler for SIGBUS/SIGSEGV that siglongjmps out to avoid either the
+// instruction being repeated infinitely (and more signals being delivered) or
+// (with SA_RESETHAND) the signal killing the process.
+namespace {
+
+void *volatile fault_address;
+sigjmp_buf jump_context;
+
+void CheckCurrentPageReadableHandler(int /*signal*/, siginfo_t *info, void *) {
+ fault_address = info->si_addr;
+
+ siglongjmp(jump_context, 1);
+}
+
+} // namespace
+void LogFileReader::CheckCurrentPageReadable() {
+ if (definitely_use_read()) return;
+
+ if (sigsetjmp(jump_context, 1) == 0) {
+ struct sigaction action;
+ action.sa_sigaction = CheckCurrentPageReadableHandler;
+ sigemptyset(&action.sa_mask);
+ action.sa_flags = SA_RESETHAND | SA_SIGINFO;
+ struct sigaction previous_bus, previous_segv;
+ if (sigaction(SIGBUS, &action, &previous_bus) == -1) {
+ PLOG(FATAL, "sigaction(SIGBUS(=%d), %p, %p) failed",
+ SIGBUS, &action, &previous_bus);
+ }
+ if (sigaction(SIGSEGV, &action, &previous_segv) == -1) {
+ PLOG(FATAL, "sigaction(SIGSEGV(=%d), %p, %p) failed",
+ SIGSEGV, &action, &previous_segv);
+ }
+
+ char __attribute__((unused)) c = current()[0];
+
+ if (sigaction(SIGBUS, &previous_bus, NULL) == -1) {
+ PLOG(FATAL, "sigaction(SIGBUS(=%d), %p, NULL) failed",
+ SIGBUS, &previous_bus);
+ }
+ if (sigaction(SIGSEGV, &previous_segv, NULL) == -1) {
+ PLOG(FATAL, "sigaction(SIGSEGV(=%d), %p, NULL) failed",
+ SIGSEGV, &previous_segv);
+ }
+ } else {
+ if (fault_address == current()) {
+ LOG(FATAL, "could not read 1 byte at offset 0x%jx into log file\n",
+ static_cast<uintmax_t>(offset()));
+ } else {
+ LOG(FATAL, "faulted at %p, not %p like we were (maybe) supposed to\n",
+ fault_address, current());
+ }
+ }
+}
+
+LogFileMessageHeader *LogFileWriter::GetWritePosition(size_t message_size) {
+ if (NeedNewPageFor(message_size)) ForceNewPage();
+ LogFileMessageHeader *const r = static_cast<LogFileMessageHeader *>(
+ static_cast<void *>(¤t()[position()]));
+ IncrementPosition(message_size);
+ return r;
+}
+
+// A number of seekable pages, not the actual file offset, is stored in *cookie.
+bool LogFileWriter::ShouldClearSeekableData(off_t *cookie,
+ size_t next_message_size) const {
+ off_t next_message_page = (offset() / kPageSize) - 1;
+ if (NeedNewPageFor(next_message_size)) {
+ ++next_message_page;
+ }
+ const off_t current_seekable_page = next_message_page / kSeekPages;
+ CHECK_LE(*cookie, current_seekable_page);
+ const bool r = *cookie != current_seekable_page;
+ *cookie = current_seekable_page;
+ return r;
+}
+
+bool LogFileWriter::NeedNewPageFor(size_t bytes) const {
+ return position() + bytes + (kAlignment - (bytes % kAlignment)) +
+ sizeof(aos_futex) >
+ kPageSize;
+}
+
+void LogFileWriter::ForceNewPage() {
+ char *const temp = current();
+ MapNextPage();
+ if (futex_set_value(
+ static_cast<aos_futex *>(static_cast<void *>(&temp[position()])),
+ 2) == -1) {
+ PLOG(WARNING, "readers will hang because futex_set_value(%p, 2) failed",
+ &temp[position()]);
+ }
+ Unmap(temp);
+}
+
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/binary_log_file.h b/aos/logging/binary_log_file.h
new file mode 100644
index 0000000..79b580d
--- /dev/null
+++ b/aos/logging/binary_log_file.h
@@ -0,0 +1,207 @@
+#ifndef AOS_LOGGING_BINARY_LOG_FILE_H_
+#define AOS_LOGGING_BINARY_LOG_FILE_H_
+
+#include <sys/types.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+
+#include "aos/logging/implementations.h"
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+
+// What to align messages to. A macro because it gets used in attributes.
+// This definition gets #undefed later. Use LogFileAccessor::kAlignment instead.
+#define MESSAGE_ALIGNMENT 8
+
+// File format: {
+// LogFileMessageHeader header;
+// char *name; // of the process that wrote the message
+// void *message;
+// } not crossing kPageSize boundaries into the file and aligned to
+// MESSAGE_ALIGNMENT.
+//
+// Field sizes designed to fit the various values from LogMessage even on
+// other machines (hopefully) because they're baked into the files. They are
+// layed out so that all of the fields are aligned even though the whole thing
+// is packed.
+//
+// A lot of the fields don't have comments because they're the same as the
+// identically named fields in LogMessage.
+struct __attribute__((aligned(MESSAGE_ALIGNMENT))) __attribute__((packed))
+ LogFileMessageHeader {
+ // Represents the type of an individual message.
+ enum class MessageType : uint16_t {
+ // char[] (no '\0' on the end).
+ kString,
+ kStructType,
+ kStruct,
+ kMatrix,
+ };
+
+ // Gets futex_set once this one has been written
+ // for readers keeping up with a live writer.
+ //
+ // Gets initialized to 0 by ftruncate.
+ //
+ // There will be something here after the last message on a "page" set to 2
+ // (by the futex_set) to indicate that the next message is on the next page.
+ aos_futex marker;
+ static_assert(sizeof(marker) == 4, "mutex changed size!");
+ static_assert(MESSAGE_ALIGNMENT >= alignof(aos_futex),
+ "MESSAGE_ALIGNMENT is too small");
+
+ uint32_t time_sec;
+ static_assert(sizeof(time_sec) >= sizeof(LogMessage::seconds),
+ "tv_sec won't fit");
+ uint32_t time_nsec;
+ static_assert(sizeof(time_nsec) >= sizeof(LogMessage::nseconds),
+ "tv_nsec won't fit");
+
+ int32_t source;
+ static_assert(sizeof(source) >= sizeof(LogMessage::source), "PIDs won't fit");
+
+ // Both including all of the bytes in that part of the message.
+ uint32_t name_size, message_size;
+
+ uint16_t sequence;
+ static_assert(sizeof(sequence) == sizeof(LogMessage::sequence),
+ "something changed");
+
+ MessageType type;
+
+ log_level level;
+ static_assert(sizeof(level) == 1, "log_level changed size!");
+};
+static_assert(std::is_pod<LogFileMessageHeader>::value,
+ "LogFileMessageHeader will to get dumped to a file");
+static_assert(offsetof(LogFileMessageHeader, marker) == 0,
+ "marker has to be at the start so readers can find it");
+
+// Handles the mmapping and munmapping for reading and writing log files.
+class LogFileAccessor {
+ public:
+ LogFileAccessor(int fd, bool writable);
+ ~LogFileAccessor() {
+ if (use_read_ == Maybe::kYes) {
+ delete[] current_;
+ }
+ }
+
+ // Asynchronously syncs all open mappings.
+ void Sync() const;
+
+ // Returns true iff we currently have the last page in the file mapped.
+ // This is fundamentally a racy question, so the return value may not be
+ // accurate by the time this method returns.
+ bool IsLastPage();
+
+ // Skips to the last page which is an even multiple of kSeekPages.
+ // This is fundamentally racy, so it may not actually be on the very last
+ // possible multiple of kSeekPages when it returns, but it should be close.
+ // This will never move backwards.
+ void SkipToLastSeekablePage();
+
+ size_t file_offset(const void *msg) {
+ return offset() + (static_cast<const char *>(msg) - current());
+ }
+
+ protected:
+ // The size of the chunks that get mmaped/munmapped together. Large enough so
+ // that not too much space is wasted and it's hopefully bigger than and a
+ // multiple of the system page size but small enough so that really large
+ // chunks of memory don't have to get mapped at the same time.
+ static const size_t kPageSize = 16384;
+ // What to align messages to, copied into an actual constant.
+ static const size_t kAlignment = MESSAGE_ALIGNMENT;
+#undef MESSAGE_ALIGNMENT
+ // Pages which are multiples of this from the beginning of a file start with
+ // no saved state (ie struct types). This allows seeking immediately to the
+ // largest currently written interval of this number when following.
+ static const size_t kSeekPages = 256;
+
+ char *current() const { return current_; }
+ size_t position() const { return position_; }
+ off_t offset() const { return offset_; }
+
+ void IncrementPosition(size_t size) {
+ position_ += size;
+ AlignPosition();
+ }
+
+ void MapNextPage();
+ void Unmap(void *location);
+
+ // Advances position to the next (aligned) location.
+ void AlignPosition() {
+ position_ += kAlignment - (position_ % kAlignment);
+ }
+
+ protected:
+ bool definitely_use_read() const { return use_read_ == Maybe::kYes; }
+ bool definitely_use_mmap() const { return use_read_ == Maybe::kNo; }
+
+ private:
+ // Used for representing things that we might know to be true/false or we
+ // might not know (yet).
+ enum class Maybe { kUnknown, kYes, kNo };
+
+ const int fd_;
+ const bool writable_;
+
+ // Into the file. Always a multiple of kPageSize.
+ off_t offset_;
+ char *current_;
+ size_t position_;
+
+ Maybe is_last_page_ = Maybe::kUnknown;
+
+ // Use read instead of mmap (necessary for fds that don't support mmap).
+ Maybe use_read_ = Maybe::kUnknown;
+};
+
+class LogFileReader : public LogFileAccessor {
+ public:
+ LogFileReader(int fd) : LogFileAccessor(fd, false) {}
+
+ // May return NULL iff wait is false.
+ const LogFileMessageHeader *ReadNextMessage(bool wait);
+
+ private:
+ // Tries reading from the current page to see if it fails because the file
+ // isn't big enough.
+ void CheckCurrentPageReadable();
+};
+
+class LogFileWriter : public LogFileAccessor {
+ public:
+ LogFileWriter(int fd) : LogFileAccessor(fd, true) {}
+
+ // message_size should be the total number of bytes needed for the message.
+ LogFileMessageHeader *GetWritePosition(size_t message_size);
+
+ // Returns true exactly once for each unique cookie on each page where cached
+ // data should be cleared.
+ // Call with a non-zero next_message_size to determine if cached data should
+ // be forgotten before writing a next_message_size-sized message.
+ // cookie should be initialized to 0.
+ bool ShouldClearSeekableData(off_t *cookie, size_t next_message_size) const;
+
+ // Forces a move to a new page for the next message.
+ // This is important when there is cacheable data that needs to be re-written
+ // before a message which will spill over onto the next page but the cacheable
+ // message being refreshed is smaller and won't get to a new page by itself.
+ void ForceNewPage();
+
+ private:
+ bool NeedNewPageFor(size_t bytes) const;
+};
+
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_BINARY_LOG_FILE_H_
diff --git a/aos/logging/binary_log_writer.cc b/aos/logging/binary_log_writer.cc
new file mode 100644
index 0000000..fb0c4c2
--- /dev/null
+++ b/aos/logging/binary_log_writer.cc
@@ -0,0 +1,324 @@
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <mntent.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+#include <string>
+
+#include <chrono>
+#include <map>
+#include <unordered_set>
+
+#include "aos/die.h"
+#include "aos/logging/binary_log_file.h"
+#include "aos/logging/implementations.h"
+#include "aos/queue_types.h"
+#include "aos/time/time.h"
+#include "aos/linux_code/configuration.h"
+#include "aos/linux_code/init.h"
+#include "aos/linux_code/ipc_lib/queue.h"
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+namespace {
+
+void CheckTypeWritten(uint32_t type_id, LogFileWriter *writer,
+ ::std::unordered_set<uint32_t> *written_type_ids) {
+ if (written_type_ids->count(type_id) > 0) return;
+ if (MessageType::IsPrimitive(type_id)) return;
+
+ const MessageType &type = type_cache::Get(type_id);
+ for (int i = 0; i < type.number_fields; ++i) {
+ CheckTypeWritten(type.fields[i]->type, writer, written_type_ids);
+ }
+
+ char buffer[1024];
+ ssize_t size = type.Serialize(buffer, sizeof(buffer));
+ if (size == -1) {
+ LOG(WARNING, "%zu-byte buffer is too small to serialize type %s\n",
+ sizeof(buffer), type.name.c_str());
+ return;
+ }
+ LogFileMessageHeader *const output =
+ writer->GetWritePosition(sizeof(LogFileMessageHeader) + size);
+
+ output->time_sec = output->time_nsec = 0;
+ output->source = getpid();
+ output->name_size = 0;
+ output->sequence = 0;
+ output->level = FATAL;
+
+ memcpy(output + 1, buffer, size);
+ output->message_size = size;
+
+ output->type = LogFileMessageHeader::MessageType::kStructType;
+ futex_set(&output->marker);
+
+ written_type_ids->insert(type_id);
+}
+
+void AllocateLogName(char **filename, const char *directory) {
+ int fileindex = 0;
+ DIR *const d = opendir(directory);
+ if (d == nullptr) {
+ PDie("could not open directory %s", directory);
+ }
+ int index = 0;
+ while (true) {
+ errno = 0;
+ struct dirent *const dir = readdir(d);
+ if (dir == nullptr) {
+ if (errno == 0) {
+ break;
+ } else {
+ PLOG(FATAL, "readdir(%p) failed", d);
+ }
+ } else {
+ if (sscanf(dir->d_name, "aos_log-%d", &index) == 1) {
+ if (index >= fileindex) {
+ fileindex = index + 1;
+ }
+ }
+ }
+ }
+ closedir(d);
+
+ char previous[512];
+ ::std::string path = ::std::string(directory) + "/aos_log-current";
+ ssize_t len = ::readlink(path.c_str(), previous, sizeof(previous));
+ if (len != -1) {
+ previous[len] = '\0';
+ } else {
+ previous[0] = '\0';
+ LOG(INFO, "Could not find aos_log-current\n");
+ printf("Could not find aos_log-current\n");
+ }
+ if (asprintf(filename, "%s/aos_log-%03d", directory, fileindex) == -1) {
+ PDie("couldn't create final name");
+ }
+ LOG(INFO, "Created log file (aos_log-%d) in directory (%s). Previous file "
+ "was (%s).\n",
+ fileindex, directory, previous);
+ printf("Created log file (aos_log-%d) in directory (%s). Previous file was "
+ "(%s).\n",
+ fileindex, directory, previous);
+}
+
+#ifdef AOS_ARCHITECTURE_arm_frc
+bool FoundThumbDrive(const char *path) {
+ FILE *mnt_fp = setmntent("/etc/mtab", "r");
+ if (mnt_fp == nullptr) {
+ Die("Could not open /etc/mtab");
+ }
+
+ bool found = false;
+ struct mntent mntbuf;
+ char buf[256];
+ while (!found) {
+ struct mntent *mount_list = getmntent_r(mnt_fp, &mntbuf, buf, sizeof(buf));
+ if (mount_list == nullptr) {
+ break;
+ }
+ if (strcmp(mount_list->mnt_dir, path) == 0) {
+ found = true;
+ }
+ }
+ endmntent(mnt_fp);
+ return found;
+}
+
+bool FindDevice(char *device, size_t device_size) {
+ char test_device[10];
+ for (char i = 'a'; i < 'z'; ++i) {
+ snprintf(test_device, sizeof(test_device), "/dev/sd%c", i);
+ LOG(INFO, "Trying to access %s\n", test_device);
+ if (access(test_device, F_OK) != -1) {
+ snprintf(device, device_size, "sd%c", i);
+ return true;
+ }
+ }
+ return false;
+}
+#endif
+
+int BinaryLogReaderMain() {
+ InitNRT();
+
+#ifdef AOS_ARCHITECTURE_arm_frc
+ char folder[128];
+
+ {
+ char dev_name[8];
+ while (!FindDevice(dev_name, sizeof(dev_name))) {
+ LOG(INFO, "Waiting for a device\n");
+ printf("Waiting for a device\n");
+ sleep(5);
+ }
+ snprintf(folder, sizeof(folder), "/media/%s1", dev_name);
+ while (!FoundThumbDrive(folder)) {
+ LOG(INFO, "Waiting for %s\n", folder);
+ printf("Waiting for %s\n", folder);
+ sleep(1);
+ }
+ snprintf(folder, sizeof(folder), "/media/%s1/", dev_name);
+ }
+
+ if (access(folder, F_OK) == -1) {
+#else
+ const char *folder = configuration::GetLoggingDirectory();
+ if (access(folder, R_OK | W_OK) == -1) {
+#endif
+ LOG(FATAL, "folder '%s' does not exist. please create it\n", folder);
+ }
+ LOG(INFO, "logging to folder '%s'\n", folder);
+
+ char *tmp;
+ AllocateLogName(&tmp, folder);
+ char *tmp2;
+ if (asprintf(&tmp2, "%s/aos_log-current", folder) == -1) {
+ PLOG(WARNING, "couldn't create current symlink name");
+ } else {
+ if (unlink(tmp2) == -1 && (errno != EROFS && errno != ENOENT)) {
+ LOG(WARNING, "unlink('%s') failed", tmp2);
+ }
+ if (symlink(tmp, tmp2) == -1) {
+ PLOG(WARNING, "symlink('%s', '%s') failed", tmp, tmp2);
+ }
+ free(tmp2);
+ }
+ int fd = open(tmp, O_SYNC | O_APPEND | O_RDWR | O_CREAT,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+ free(tmp);
+ if (fd == -1) {
+ PLOG(FATAL, "opening file '%s' failed", tmp);
+ }
+ LogFileWriter writer(fd);
+
+ RawQueue *queue = GetLoggingQueue();
+
+ ::std::unordered_set<uint32_t> written_type_ids;
+ off_t clear_type_ids_cookie = 0;
+
+ while (true) {
+ const LogMessage *const msg =
+ static_cast<const LogMessage *>(queue->ReadMessage(RawQueue::kNonBlock));
+ if (msg == NULL) {
+ // If we've emptied the queue, then wait for a bit before starting to read
+ // again so the queue can buffer up some logs. This avoids lots of context
+ // switches and mutex contention which happens if we're constantly reading
+ // new messages as they come in.
+ ::std::this_thread::sleep_for(::std::chrono::milliseconds(100));
+ continue;
+ }
+
+ const size_t raw_output_length =
+ sizeof(LogFileMessageHeader) + msg->name_length + msg->message_length;
+ size_t output_length = raw_output_length;
+ if (msg->type == LogMessage::Type::kStruct) {
+ output_length += sizeof(msg->structure.type_id) + sizeof(uint32_t) +
+ msg->structure.string_length;
+ if (writer.ShouldClearSeekableData(&clear_type_ids_cookie,
+ output_length)) {
+ writer.ForceNewPage();
+ written_type_ids.clear();
+ }
+ CheckTypeWritten(msg->structure.type_id, &writer, &written_type_ids);
+ } else if (msg->type == LogMessage::Type::kMatrix) {
+ output_length +=
+ sizeof(msg->matrix.type) + sizeof(uint32_t) + sizeof(uint16_t) +
+ sizeof(uint16_t) + msg->matrix.string_length;
+ CHECK(MessageType::IsPrimitive(msg->matrix.type));
+ }
+ LogFileMessageHeader *const output = writer.GetWritePosition(output_length);
+ char *output_strings = reinterpret_cast<char *>(output) + sizeof(*output);
+ output->name_size = msg->name_length;
+ output->message_size = msg->message_length;
+ output->source = msg->source;
+ output->level = msg->level;
+ output->time_sec = msg->seconds;
+ output->time_nsec = msg->nseconds;
+ output->sequence = msg->sequence;
+ memcpy(output_strings, msg->name, msg->name_length);
+
+ switch (msg->type) {
+ case LogMessage::Type::kString:
+ memcpy(output_strings + msg->name_length, msg->message,
+ msg->message_length);
+ output->type = LogFileMessageHeader::MessageType::kString;
+ break;
+ case LogMessage::Type::kStruct: {
+ char *position = output_strings + msg->name_length;
+
+ memcpy(position, &msg->structure.type_id,
+ sizeof(msg->structure.type_id));
+ position += sizeof(msg->structure.type_id);
+ output->message_size += sizeof(msg->structure.type_id);
+
+ const uint32_t length = msg->structure.string_length;
+ memcpy(position, &length, sizeof(length));
+ position += sizeof(length);
+ memcpy(position, msg->structure.serialized,
+ length + msg->message_length);
+ position += length + msg->message_length;
+ output->message_size += sizeof(length) + length;
+
+ output->type = LogFileMessageHeader::MessageType::kStruct;
+ } break;
+ case LogMessage::Type::kMatrix: {
+ char *position = output_strings + msg->name_length;
+
+ memcpy(position, &msg->matrix.type, sizeof(msg->matrix.type));
+ position += sizeof(msg->matrix.type);
+ output->message_size += sizeof(msg->matrix.type);
+
+ uint32_t length = msg->matrix.string_length;
+ memcpy(position, &length, sizeof(length));
+ position += sizeof(length);
+ output->message_size += sizeof(length);
+
+ uint16_t rows = msg->matrix.rows, cols = msg->matrix.cols;
+ memcpy(position, &rows, sizeof(rows));
+ position += sizeof(rows);
+ memcpy(position, &cols, sizeof(cols));
+ position += sizeof(cols);
+ output->message_size += sizeof(rows) + sizeof(cols);
+ CHECK_EQ(msg->message_length,
+ MessageType::Sizeof(msg->matrix.type) * rows * cols);
+
+ memcpy(position, msg->matrix.data, msg->message_length + length);
+ output->message_size += length;
+
+ output->type = LogFileMessageHeader::MessageType::kMatrix;
+ } break;
+ }
+
+ if (output->message_size - msg->message_length !=
+ output_length - raw_output_length) {
+ LOG(FATAL, "%zu != %zu\n", output->message_size - msg->message_length,
+ output_length - raw_output_length);
+ }
+
+ futex_set(&output->marker);
+
+ queue->FreeMessage(msg);
+ }
+
+ Cleanup();
+ return 0;
+}
+
+} // namespace
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
+
+int main() {
+ return ::aos::logging::linux_code::BinaryLogReaderMain();
+}
diff --git a/aos/logging/context.cc b/aos/logging/context.cc
new file mode 100644
index 0000000..3701b00
--- /dev/null
+++ b/aos/logging/context.cc
@@ -0,0 +1,98 @@
+#include "aos/logging/context.h"
+
+#include <string.h>
+#include <sys/prctl.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <string>
+
+#include "aos/die.h"
+#include "aos/linux_code/complex_thread_local.h"
+
+namespace aos {
+namespace logging {
+namespace internal {
+namespace {
+
+// TODO(brians): Differentiate between threads with the same name in the same
+// process.
+
+::std::string GetMyName() {
+ // The maximum number of characters that can make up a thread name.
+ // The docs are unclear if it can be 16 characters with no '\0', so we'll be
+ // safe by adding our own where necessary.
+ static const size_t kThreadNameLength = 16;
+
+ ::std::string process_name(program_invocation_short_name);
+
+ char thread_name_array[kThreadNameLength + 1];
+ if (prctl(PR_GET_NAME, thread_name_array) != 0) {
+ PDie("prctl(PR_GET_NAME, %p) failed", thread_name_array);
+ }
+ thread_name_array[sizeof(thread_name_array) - 1] = '\0';
+ ::std::string thread_name(thread_name_array);
+
+ // If the first bunch of characters are the same.
+ // We cut off comparing at the shorter of the 2 strings because one or the
+ // other often ends up cut off.
+ if (strncmp(thread_name.c_str(), process_name.c_str(),
+ ::std::min(thread_name.length(), process_name.length())) == 0) {
+ // This thread doesn't have an actual name.
+ return process_name;
+ }
+
+ return process_name + '.' + thread_name;
+}
+
+::aos::ComplexThreadLocal<Context> my_context;
+
+// True if we're going to delete the current Context object ASAP. The
+// reason for doing this instead of just deleting them is that tsan (at least)
+// doesn't like it when pthread_atfork handlers do complicated stuff and it's
+// not a great idea anyways.
+thread_local bool delete_current_context(false);
+
+} // namespace
+
+::std::atomic<LogImplementation *> global_top_implementation(NULL);
+
+Context::Context()
+ : implementation(global_top_implementation.load()),
+ sequence(0) {
+ cork_data.Reset();
+}
+
+// Used in aos/linux_code/init.cc when a thread's name is changed.
+void ReloadThreadName() {
+ if (my_context.created()) {
+ ::std::string my_name = GetMyName();
+ if (my_name.size() + 1 > sizeof(Context::name)) {
+ Die("logging: process/thread name '%s' is too long\n",
+ my_name.c_str());
+ }
+ strcpy(my_context->name, my_name.c_str());
+ my_context->name_size = my_name.size();
+ }
+}
+
+Context *Context::Get() {
+ if (__builtin_expect(delete_current_context, false)) {
+ my_context.Clear();
+ delete_current_context = false;
+ }
+ if (__builtin_expect(!my_context.created(), false)) {
+ my_context.Create();
+ ReloadThreadName();
+ my_context->source = getpid();
+ }
+ return my_context.get();
+}
+
+void Context::Delete() {
+ delete_current_context = true;
+}
+
+} // namespace internal
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/context.h b/aos/logging/context.h
new file mode 100644
index 0000000..b2abc0d
--- /dev/null
+++ b/aos/logging/context.h
@@ -0,0 +1,84 @@
+#ifndef AOS_LOGGING_CONTEXT_H_
+#define AOS_LOGGING_CONTEXT_H_
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <sys/types.h>
+#include <limits.h>
+
+#include <atomic>
+
+#include "aos/logging/sizes.h"
+
+namespace aos {
+namespace logging {
+
+class LogImplementation;
+
+// This is where all of the code that is only used by actual LogImplementations
+// goes.
+namespace internal {
+
+extern ::std::atomic<LogImplementation *> global_top_implementation;
+
+// An separate instance of this class is accessible from each task/thread.
+// NOTE: It will get deleted in the child of a fork.
+//
+// Get() and Delete() are implemented in the platform-specific interface.cc
+// file.
+struct Context {
+ Context();
+
+ // Gets the Context object for this task/thread. Will create one the first
+ // time it is called.
+ //
+ // The implementation for each platform will lazily instantiate a new instance
+ // and then initialize name the first time.
+ // IMPORTANT: The implementation of this can not use logging.
+ static Context *Get();
+ // Deletes the Context object for this task/thread so that the next Get() is
+ // called it will create a new one.
+ // It is valid to call this when Get() has never been called.
+ // This also gets called after a fork(2) in the new process, where it should
+ // still work to clean up any state.
+ static void Delete();
+
+ // Which one to log to right now.
+ // Will be NULL if there is no logging implementation to use right now.
+ LogImplementation *implementation;
+
+ // A name representing this task/(process and thread).
+ char name[LOG_MESSAGE_NAME_LEN];
+ size_t name_size;
+
+ // What to assign LogMessage::source to in this task/thread.
+ pid_t source;
+
+ // The sequence value to send out with the next message.
+ uint16_t sequence;
+
+ // Contains all of the information related to implementing LOG_CORK and
+ // LOG_UNCORK.
+ struct {
+ char message[LOG_MESSAGE_LEN];
+ int line_min, line_max;
+ // Sets the data up to record a new series of corked logs.
+ void Reset() {
+ message[0] = '\0'; // make strlen of it 0
+ line_min = INT_MAX;
+ line_max = -1;
+ function = NULL;
+ }
+ // The function that the calls are in.
+ // REMEMBER: While the compiler/linker will probably optimize all of the
+ // identical strings to point to the same data, it might not, so using == to
+ // compare this with another value is a bad idea.
+ const char *function;
+ } cork_data;
+};
+
+} // namespace internal
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_CONTEXT_H_
diff --git a/aos/logging/implementations.cc b/aos/logging/implementations.cc
new file mode 100644
index 0000000..337aa8f
--- /dev/null
+++ b/aos/logging/implementations.cc
@@ -0,0 +1,437 @@
+#include "aos/logging/implementations.h"
+
+#include <stdarg.h>
+#include <inttypes.h>
+
+#include <algorithm>
+#include <chrono>
+
+#include "aos/die.h"
+#include "aos/logging/printf_formats.h"
+#include "aos/queue_types.h"
+#include "aos/time/time.h"
+#include "aos/linux_code/ipc_lib/queue.h"
+#include "aos/once.h"
+
+namespace aos {
+namespace logging {
+namespace {
+
+namespace chrono = ::std::chrono;
+
+// The root LogImplementation. It only logs to stderr/stdout.
+// Some of the things specified in the LogImplementation documentation doesn't
+// apply here (mostly the parts about being able to use LOG) because this is the
+// root one.
+class RootLogImplementation : public SimpleLogImplementation {
+ public:
+ void have_other_implementation() { only_implementation_ = false; }
+
+ private:
+ void set_next(LogImplementation *) override {
+ LOG(FATAL, "can't have a next logger from here\n");
+ }
+
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)))
+ void DoLog(log_level level, const char *format, va_list ap) override {
+ LogMessage message;
+ internal::FillInMessage(level, format, ap, &message);
+ internal::PrintMessage(stderr, message);
+ if (!only_implementation_) {
+ fputs("root logger got used, see stderr for message\n", stdout);
+ }
+ }
+
+ bool only_implementation_ = true;
+};
+
+RootLogImplementation *root_implementation = nullptr;
+
+void SetGlobalImplementation(LogImplementation *implementation) {
+ if (root_implementation == nullptr) {
+ fputs("Somebody didn't call logging::Init()!\n", stderr);
+ abort();
+ }
+
+ internal::Context *context = internal::Context::Get();
+
+ context->implementation = implementation;
+ internal::global_top_implementation.store(implementation);
+}
+
+void NewContext() {
+ internal::Context::Delete();
+}
+
+void *DoInit() {
+ SetGlobalImplementation(root_implementation = new RootLogImplementation());
+
+ if (pthread_atfork(NULL /*prepare*/, NULL /*parent*/,
+ NewContext /*child*/) != 0) {
+ LOG(FATAL, "pthread_atfork(NULL, NULL, %p) failed\n",
+ NewContext);
+ }
+
+ return NULL;
+}
+
+} // namespace
+namespace internal {
+namespace {
+
+void FillInMessageBase(log_level level, LogMessage *message) {
+ Context *context = Context::Get();
+
+ message->level = level;
+ message->source = context->source;
+ memcpy(message->name, context->name, context->name_size);
+ message->name_length = context->name_size;
+
+ monotonic_clock::time_point monotonic_now = monotonic_clock::now();
+ message->seconds =
+ chrono::duration_cast<chrono::seconds>(monotonic_now.time_since_epoch())
+ .count();
+ message->nseconds =
+ chrono::duration_cast<chrono::nanoseconds>(
+ monotonic_now.time_since_epoch() - chrono::seconds(message->seconds))
+ .count();
+
+ message->sequence = context->sequence++;
+}
+
+} // namespace
+
+void FillInMessageStructure(log_level level,
+ const ::std::string &message_string, size_t size,
+ const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize,
+ LogMessage *message) {
+ type_cache::AddShm(type->id);
+ message->structure.type_id = type->id;
+
+ FillInMessageBase(level, message);
+
+ if (message_string.size() + size > sizeof(message->structure.serialized)) {
+ LOG(FATAL, "serialized struct %s (size %zd) and message %s too big\n",
+ type->name.c_str(), size, message_string.c_str());
+ }
+ message->structure.string_length = message_string.size();
+ memcpy(message->structure.serialized, message_string.data(),
+ message->structure.string_length);
+
+ message->message_length = serialize(
+ &message->structure.serialized[message->structure.string_length]);
+ message->type = LogMessage::Type::kStruct;
+}
+
+void FillInMessageMatrix(log_level level,
+ const ::std::string &message_string, uint32_t type_id,
+ int rows, int cols, const void *data,
+ LogMessage *message) {
+ CHECK(MessageType::IsPrimitive(type_id));
+ message->matrix.type = type_id;
+
+ const auto element_size = MessageType::Sizeof(type_id);
+
+ FillInMessageBase(level, message);
+
+ message->message_length = rows * cols * element_size;
+ if (message_string.size() + message->message_length >
+ sizeof(message->matrix.data)) {
+ LOG(FATAL, "%dx%d matrix of type %" PRIu32
+ " (size %u) and message %s is too big\n",
+ rows, cols, type_id, element_size, message_string.c_str());
+ }
+ message->matrix.string_length = message_string.size();
+ memcpy(message->matrix.data, message_string.data(),
+ message->matrix.string_length);
+
+ message->matrix.rows = rows;
+ message->matrix.cols = cols;
+ SerializeMatrix(type_id, &message->matrix.data[message->matrix.string_length],
+ data, rows, cols);
+ message->type = LogMessage::Type::kMatrix;
+}
+
+void FillInMessage(log_level level, const char *format, va_list ap,
+ LogMessage *message) {
+ FillInMessageBase(level, message);
+
+ message->message_length =
+ ExecuteFormat(message->message, sizeof(message->message), format, ap);
+ message->type = LogMessage::Type::kString;
+}
+
+void PrintMessage(FILE *output, const LogMessage &message) {
+#define BASE_ARGS \
+ AOS_LOGGING_BASE_ARGS( \
+ message.name_length, message.name, static_cast<int32_t>(message.source), \
+ message.sequence, message.level, message.seconds, message.nseconds)
+ switch (message.type) {
+ case LogMessage::Type::kString:
+ fprintf(output, AOS_LOGGING_BASE_FORMAT "%.*s", BASE_ARGS,
+ static_cast<int>(message.message_length), message.message);
+ break;
+ case LogMessage::Type::kStruct: {
+ char buffer[4096];
+ size_t output_length = sizeof(buffer);
+ size_t input_length = message.message_length;
+ if (!PrintMessage(
+ buffer, &output_length,
+ message.structure.serialized + message.structure.string_length,
+ &input_length, type_cache::Get(message.structure.type_id))) {
+ LOG(FATAL,
+ "printing message (%.*s) of type %s into %zu-byte buffer failed\n",
+ static_cast<int>(message.message_length), message.message,
+ type_cache::Get(message.structure.type_id).name.c_str(),
+ sizeof(buffer));
+ }
+ if (input_length > 0) {
+ LOG(WARNING, "%zu extra bytes on message of type %s\n", input_length,
+ type_cache::Get(message.structure.type_id).name.c_str());
+ }
+ fprintf(output, AOS_LOGGING_BASE_FORMAT "%.*s: %.*s\n", BASE_ARGS,
+ static_cast<int>(message.structure.string_length),
+ message.structure.serialized,
+ static_cast<int>(sizeof(buffer) - output_length), buffer);
+ } break;
+ case LogMessage::Type::kMatrix: {
+ char buffer[1024];
+ size_t output_length = sizeof(buffer);
+ if (message.message_length !=
+ static_cast<size_t>(message.matrix.rows * message.matrix.cols *
+ MessageType::Sizeof(message.matrix.type))) {
+ LOG(FATAL, "expected %d bytes of matrix data but have %zu\n",
+ message.matrix.rows * message.matrix.cols *
+ MessageType::Sizeof(message.matrix.type),
+ message.message_length);
+ }
+ if (!PrintMatrix(buffer, &output_length,
+ message.matrix.data + message.matrix.string_length,
+ message.matrix.type, message.matrix.rows,
+ message.matrix.cols)) {
+ LOG(FATAL, "printing %dx%d matrix of type %" PRIu32 " failed\n",
+ message.matrix.rows, message.matrix.cols, message.matrix.type);
+ }
+ fprintf(output, AOS_LOGGING_BASE_FORMAT "%.*s: %.*s\n", BASE_ARGS,
+ static_cast<int>(message.matrix.string_length),
+ message.matrix.data,
+ static_cast<int>(sizeof(buffer) - output_length), buffer);
+ } break;
+ }
+#undef BASE_ARGS
+}
+
+} // namespace internal
+
+void SimpleLogImplementation::LogStruct(
+ log_level level, const ::std::string &message, size_t size,
+ const MessageType *type, const ::std::function<size_t(char *)> &serialize) {
+ char serialized[1024];
+ if (size > sizeof(serialized)) {
+ LOG(FATAL, "structure of type %s too big to serialize\n",
+ type->name.c_str());
+ }
+ size_t used = serialize(serialized);
+ char printed[1024];
+ size_t printed_bytes = sizeof(printed);
+ if (!PrintMessage(printed, &printed_bytes, serialized, &used, *type)) {
+ LOG(FATAL, "PrintMessage(%p, %p(=%zd), %p, %p(=%zd), %p(name=%s)) failed\n",
+ printed, &printed_bytes, printed_bytes, serialized, &used, used, type,
+ type->name.c_str());
+ }
+ DoLogVariadic(level, "%.*s: %.*s\n", static_cast<int>(message.size()),
+ message.data(),
+ static_cast<int>(sizeof(printed) - printed_bytes), printed);
+}
+
+void SimpleLogImplementation::LogMatrix(
+ log_level level, const ::std::string &message, uint32_t type_id,
+ int rows, int cols, const void *data) {
+ char serialized[1024];
+ if (static_cast<size_t>(rows * cols * MessageType::Sizeof(type_id)) >
+ sizeof(serialized)) {
+ LOG(FATAL, "matrix of size %u too big to serialize\n",
+ rows * cols * MessageType::Sizeof(type_id));
+ }
+ SerializeMatrix(type_id, serialized, data, rows, cols);
+ char printed[1024];
+ size_t printed_bytes = sizeof(printed);
+ if (!PrintMatrix(printed, &printed_bytes, serialized, type_id, rows, cols)) {
+ LOG(FATAL, "PrintMatrix(%p, %p(=%zd), %p, %" PRIu32 ", %d, %d) failed\n",
+ printed, &printed_bytes, printed_bytes, serialized, type_id, rows,
+ cols);
+ }
+ DoLogVariadic(level, "%.*s: %.*s\n", static_cast<int>(message.size()),
+ message.data(),
+ static_cast<int>(sizeof(printed) - printed_bytes), printed);
+}
+
+void HandleMessageLogImplementation::DoLog(log_level level, const char *format,
+ va_list ap) {
+ LogMessage message;
+ internal::FillInMessage(level, format, ap, &message);
+ HandleMessage(message);
+}
+
+void HandleMessageLogImplementation::LogStruct(
+ log_level level, const ::std::string &message_string, size_t size,
+ const MessageType *type, const ::std::function<size_t(char *)> &serialize) {
+ LogMessage message;
+ internal::FillInMessageStructure(level, message_string, size, type, serialize,
+ &message);
+ HandleMessage(message);
+}
+
+void HandleMessageLogImplementation::LogMatrix(
+ log_level level, const ::std::string &message_string, uint32_t type_id,
+ int rows, int cols, const void *data) {
+ LogMessage message;
+ internal::FillInMessageMatrix(level, message_string, type_id, rows, cols,
+ data, &message);
+ HandleMessage(message);
+}
+
+StreamLogImplementation::StreamLogImplementation(FILE *stream)
+ : stream_(stream) {}
+
+void StreamLogImplementation::HandleMessage(const LogMessage &message) {
+ internal::PrintMessage(stream_, message);
+}
+
+void AddImplementation(LogImplementation *implementation) {
+ internal::Context *context = internal::Context::Get();
+
+ if (implementation->next() != NULL) {
+ LOG(FATAL, "%p already has a next implementation, but it's not"
+ " being used yet\n", implementation);
+ }
+
+ LogImplementation *old = context->implementation;
+ if (old != NULL) {
+ implementation->set_next(old);
+ }
+ SetGlobalImplementation(implementation);
+ root_implementation->have_other_implementation();
+}
+
+void Init() {
+ static Once<void> once(DoInit);
+ once.Get();
+}
+
+void Load() {
+ internal::Context::Get();
+}
+
+void Cleanup() {
+ internal::Context::Delete();
+}
+
+namespace {
+
+RawQueue *queue = NULL;
+
+int dropped_messages = 0;
+monotonic_clock::time_point dropped_start, backoff_start;
+// Wait this long after dropping a message before even trying to write any more.
+constexpr chrono::milliseconds kDropBackoff = chrono::milliseconds(100);
+
+LogMessage *GetMessageOrDie() {
+ LogMessage *message = static_cast<LogMessage *>(queue->GetMessage());
+ if (message == NULL) {
+ LOG(FATAL, "%p->GetMessage() failed\n", queue);
+ } else {
+ return message;
+ }
+}
+
+void Write(LogMessage *msg) {
+ if (__builtin_expect(dropped_messages > 0, false)) {
+ monotonic_clock::time_point message_time(
+ chrono::seconds(msg->seconds) + chrono::nanoseconds(msg->nseconds));
+ if (message_time - backoff_start < kDropBackoff) {
+ ++dropped_messages;
+ queue->FreeMessage(msg);
+ return;
+ }
+
+ LogMessage *dropped_message = GetMessageOrDie();
+ chrono::seconds dropped_start_sec = chrono::duration_cast<chrono::seconds>(
+ dropped_start.time_since_epoch());
+ chrono::nanoseconds dropped_start_nsec =
+ chrono::duration_cast<chrono::nanoseconds>(
+ dropped_start.time_since_epoch() - dropped_start_sec);
+ internal::FillInMessageVarargs(
+ ERROR, dropped_message,
+ "%d logs starting at %" PRId32 ".%" PRId32 " dropped\n",
+ dropped_messages, static_cast<int32_t>(dropped_start_sec.count()),
+ static_cast<int32_t>(dropped_start_nsec.count()));
+ if (queue->WriteMessage(dropped_message, RawQueue::kNonBlock)) {
+ dropped_messages = 0;
+ } else {
+ // Don't even bother trying to write this message because it's not likely
+ // to work and it would be confusing to have one log in the middle of a
+ // string of failures get through.
+ ++dropped_messages;
+ backoff_start = message_time;
+ queue->FreeMessage(msg);
+ return;
+ }
+ }
+ if (!queue->WriteMessage(msg, RawQueue::kNonBlock)) {
+ if (dropped_messages == 0) {
+ monotonic_clock::time_point message_time(
+ chrono::seconds(msg->seconds) + chrono::nanoseconds(msg->nseconds));
+ dropped_start = backoff_start = message_time;
+ }
+ ++dropped_messages;
+ }
+}
+
+class LinuxQueueLogImplementation : public LogImplementation {
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)))
+ void DoLog(log_level level, const char *format, va_list ap) override {
+ LogMessage *message = GetMessageOrDie();
+ internal::FillInMessage(level, format, ap, message);
+ Write(message);
+ }
+
+ void LogStruct(log_level level, const ::std::string &message_string,
+ size_t size, const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize) override {
+ LogMessage *message = GetMessageOrDie();
+ internal::FillInMessageStructure(level, message_string, size, type,
+ serialize, message);
+ Write(message);
+ }
+
+ void LogMatrix(log_level level, const ::std::string &message_string,
+ uint32_t type_id, int rows, int cols,
+ const void *data) override {
+ LogMessage *message = GetMessageOrDie();
+ internal::FillInMessageMatrix(level, message_string, type_id, rows, cols,
+ data, message);
+ Write(message);
+ }
+};
+
+} // namespace
+
+RawQueue *GetLoggingQueue() {
+ return RawQueue::Fetch("LoggingQueue", sizeof(LogMessage), 1323, 40000);
+}
+
+void RegisterQueueImplementation() {
+ Init();
+
+ queue = GetLoggingQueue();
+ if (queue == NULL) {
+ Die("logging: couldn't fetch queue\n");
+ }
+
+ AddImplementation(new LinuxQueueLogImplementation());
+}
+
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/implementations.h b/aos/logging/implementations.h
new file mode 100644
index 0000000..69a805b
--- /dev/null
+++ b/aos/logging/implementations.h
@@ -0,0 +1,215 @@
+#ifndef AOS_LOGGING_IMPLEMENTATIONS_H_
+#define AOS_LOGGING_IMPLEMENTATIONS_H_
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+#include <string>
+#include <functional>
+#include <atomic>
+
+#include "aos/logging/context.h"
+#include "aos/logging/interface.h"
+#include "aos/logging/logging.h"
+#include "aos/logging/sizes.h"
+#include "aos/macros.h"
+#include "aos/mutex/mutex.h"
+#include "aos/type_traits/type_traits.h"
+#include "aos/once.h"
+
+namespace aos {
+
+struct MessageType;
+class RawQueue;
+
+} // namespace aos
+
+// This file has various concrete LogImplementations.
+
+namespace aos {
+namespace logging {
+
+// Unless explicitly stated otherwise, format must always be a string constant,
+// args are printf-style arguments for format, and ap is a va_list of args.
+// The validity of format and args together will be checked at compile time
+// using a function attribute.
+
+// Contains all of the information about a given logging call.
+struct LogMessage {
+ enum class Type : uint8_t {
+ kString, kStruct, kMatrix
+ };
+
+ int32_t seconds, nseconds;
+ // message_length is just the length of the actual data (which member depends
+ // on the type).
+ size_t message_length, name_length;
+ pid_t source;
+ static_assert(sizeof(source) == 4, "that's how they get printed");
+ // Per task/thread.
+ uint16_t sequence;
+ Type type;
+ log_level level;
+ char name[LOG_MESSAGE_NAME_LEN];
+ union {
+ char message[LOG_MESSAGE_LEN];
+ struct {
+ uint32_t type_id;
+ size_t string_length;
+ // The message string and then the serialized structure.
+ char serialized[LOG_MESSAGE_LEN - sizeof(type) - sizeof(string_length)];
+ } structure;
+ struct {
+ // The type ID of the element type.
+ uint32_t type;
+ int rows, cols;
+ size_t string_length;
+ // The message string and then the serialized matrix.
+ char
+ data[LOG_MESSAGE_LEN - sizeof(type) - sizeof(rows) - sizeof(cols)];
+ } matrix;
+ };
+};
+static_assert(shm_ok<LogMessage>::value, "it's going in a queue");
+
+// Returns left > right. LOG_UNKNOWN is most important.
+static inline bool log_gt_important(log_level left, log_level right) {
+ if (left == ERROR) left = 3;
+ if (right == ERROR) right = 3;
+ return left > right;
+}
+
+// Returns a string representing level or "unknown".
+static inline const char *log_str(log_level level) {
+#define DECL_LEVEL(name, value) if (level == name) return #name;
+ DECL_LEVELS;
+#undef DECL_LEVEL
+ return "unknown";
+}
+// Returns the log level represented by str or LOG_UNKNOWN.
+static inline log_level str_log(const char *str) {
+#define DECL_LEVEL(name, value) if (!strcmp(str, #name)) return name;
+ DECL_LEVELS;
+#undef DECL_LEVEL
+ return LOG_UNKNOWN;
+}
+
+// A LogImplementation where LogStruct and LogMatrix just create a string with
+// PrintMessage and then forward on to DoLog.
+class SimpleLogImplementation : public LogImplementation {
+ private:
+ void LogStruct(log_level level, const ::std::string &message, size_t size,
+ const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize) override;
+ void LogMatrix(log_level level, const ::std::string &message,
+ uint32_t type_id, int rows, int cols,
+ const void *data) override;
+};
+
+// Implements all of the DoLog* methods in terms of a (pure virtual in this
+// class) HandleMessage method that takes a pointer to the message.
+class HandleMessageLogImplementation : public LogImplementation {
+ private:
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)))
+ void DoLog(log_level level, const char *format, va_list ap) override;
+ void LogStruct(log_level level, const ::std::string &message_string,
+ size_t size, const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize) override;
+ void LogMatrix(log_level level, const ::std::string &message_string,
+ uint32_t type_id, int rows, int cols,
+ const void *data) override;
+
+ virtual void HandleMessage(const LogMessage &message) = 0;
+};
+
+// A log implementation that dumps all messages to a C stdio stream.
+class StreamLogImplementation : public HandleMessageLogImplementation {
+ public:
+ StreamLogImplementation(FILE *stream);
+
+ private:
+ void HandleMessage(const LogMessage &message) override;
+
+ FILE *const stream_;
+};
+
+// Adds another implementation to the stack of implementations in this
+// task/thread.
+// Any tasks/threads created after this call will also use this implementation.
+// The cutoff is when the state in a given task/thread is created (either lazily
+// when needed or by calling Load()).
+// The logging system takes ownership of implementation. It will delete it if
+// necessary, so it must be created with new.
+void AddImplementation(LogImplementation *implementation);
+
+// Must be called at least once per process/load before anything else is
+// called. This function is safe to call multiple times from multiple
+// tasks/threads.
+void Init();
+
+// Forces all of the state that is usually lazily created when first needed to
+// be created when called. Cleanup() will delete it.
+void Load();
+
+// Resets all information in this task/thread to its initial state.
+// NOTE: This is not the opposite of Init(). The state that this deletes is
+// lazily created when needed. It is actually the opposite of Load().
+void Cleanup();
+
+// Returns a queue which deals with LogMessage-sized messages.
+// The caller takes ownership.
+RawQueue *GetLoggingQueue();
+
+// Calls AddImplementation to register the standard linux logging implementation
+// which sends the messages through a queue. This implementation relies on
+// another process(es) to read the log messages that it puts into the queue.
+// This function is usually called by aos::Init*.
+void RegisterQueueImplementation();
+
+// This is where all of the code that is only used by actual LogImplementations
+// goes.
+namespace internal {
+
+// Fills in all the parts of message according to the given inputs (with type
+// kStruct).
+void FillInMessageStructure(log_level level,
+ const ::std::string &message_string, size_t size,
+ const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize,
+ LogMessage *message);
+
+// Fills in all the parts of the message according to the given inputs (with
+// type kMatrix).
+void FillInMessageMatrix(log_level level,
+ const ::std::string &message_string, uint32_t type_id,
+ int rows, int cols, const void *data,
+ LogMessage *message);
+
+// Fills in *message according to the given inputs (with type kString).
+// Used for implementing LogImplementation::DoLog.
+void FillInMessage(log_level level, const char *format, va_list ap,
+ LogMessage *message)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 2, 0)));
+
+__attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 4)))
+static inline void FillInMessageVarargs(log_level level, LogMessage *message,
+ const char *format, ...) {
+ va_list ap;
+ va_start(ap, format);
+ FillInMessage(level, format, ap, message);
+ va_end(ap);
+}
+
+// Prints message to output.
+void PrintMessage(FILE *output, const LogMessage &message);
+
+} // namespace internal
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_IMPLEMENTATIONS_H_
diff --git a/aos/logging/implementations_test.cc b/aos/logging/implementations_test.cc
new file mode 100644
index 0000000..ef69658
--- /dev/null
+++ b/aos/logging/implementations_test.cc
@@ -0,0 +1,249 @@
+#include <inttypes.h>
+
+#include <chrono>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "aos/logging/implementations.h"
+#include "aos/time/time.h"
+#include "aos/logging/printf_formats.h"
+
+using ::testing::AssertionResult;
+using ::testing::AssertionSuccess;
+using ::testing::AssertionFailure;
+
+namespace aos {
+namespace logging {
+namespace testing {
+
+namespace chrono = ::std::chrono;
+
+class TestLogImplementation : public SimpleLogImplementation {
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)))
+ void DoLog(log_level level, const char *format, va_list ap) override {
+ internal::FillInMessage(level, format, ap, &message_);
+
+ if (level == FATAL) {
+ internal::PrintMessage(stderr, message_);
+ abort();
+ }
+
+ used_ = true;
+ }
+
+ LogMessage message_;
+
+ public:
+ const LogMessage &message() { return message_; }
+ bool used() { return used_; }
+ void reset_used() { used_ = false; }
+
+ TestLogImplementation() : used_(false) {}
+
+ bool used_;
+};
+class LoggingTest : public ::testing::Test {
+ protected:
+ AssertionResult WasAnythingLogged() {
+ if (log_implementation->used()) {
+ return AssertionSuccess() << "read message '" <<
+ log_implementation->message().message << "'";
+ }
+ return AssertionFailure();
+ }
+ AssertionResult WasLogged(log_level level, const std::string message) {
+ if (!log_implementation->used()) {
+ return AssertionFailure() << "nothing was logged";
+ }
+ if (log_implementation->message().level != level) {
+ return AssertionFailure() << "a message with level " <<
+ log_str(log_implementation->message().level) <<
+ " was logged instead of " << log_str(level);
+ }
+ internal::Context *context = internal::Context::Get();
+ if (log_implementation->message().source != context->source) {
+ LOG(FATAL, "got a message from %" PRIu32 ", but we're %" PRIu32 "\n",
+ static_cast<uint32_t>(log_implementation->message().source),
+ static_cast<uint32_t>(context->source));
+ }
+ if (log_implementation->message().name_length != context->name_size ||
+ memcmp(log_implementation->message().name, context->name,
+ context->name_size) !=
+ 0) {
+ LOG(FATAL, "got a message from %.*s, but we're %s\n",
+ static_cast<int>(log_implementation->message().name_length),
+ log_implementation->message().name, context->name);
+ }
+ if (strstr(log_implementation->message().message, message.c_str())
+ == NULL) {
+ return AssertionFailure() << "got a message of '" <<
+ log_implementation->message().message <<
+ "' but expected it to contain '" << message << "'";
+ }
+
+ return AssertionSuccess() << log_implementation->message().message;
+ }
+
+ private:
+ void SetUp() override {
+ static bool first = true;
+ if (first) {
+ first = false;
+
+ Init();
+ AddImplementation(log_implementation = new TestLogImplementation());
+ }
+
+ log_implementation->reset_used();
+ }
+ void TearDown() override {
+ Cleanup();
+ }
+
+ static TestLogImplementation *log_implementation;
+};
+TestLogImplementation *LoggingTest::log_implementation(NULL);
+typedef LoggingTest LoggingDeathTest;
+
+// Tests both basic logging functionality and that the test setup works
+// correctly.
+TEST_F(LoggingTest, Basic) {
+ EXPECT_FALSE(WasAnythingLogged());
+ LOG(INFO, "test log 1\n");
+ EXPECT_TRUE(WasLogged(INFO, "test log 1\n"));
+ LOG(INFO, "test log 1.5\n");
+ // there's a subtle typo on purpose...
+ EXPECT_FALSE(WasLogged(INFO, "test log 15\n"));
+ LOG(ERROR, "test log 2=%d\n", 55);
+ EXPECT_TRUE(WasLogged(ERROR, "test log 2=55\n"));
+ LOG(ERROR, "test log 3\n");
+ EXPECT_FALSE(WasLogged(WARNING, "test log 3\n"));
+ LOG(WARNING, "test log 4\n");
+ EXPECT_TRUE(WasAnythingLogged());
+}
+TEST_F(LoggingTest, Cork) {
+ static const int begin_line = __LINE__;
+ LOG_CORK("first part ");
+ LOG_CORK("second part (=%d) ", 19);
+ EXPECT_FALSE(WasAnythingLogged());
+ LOG_CORK("third part ");
+ static const int end_line = __LINE__;
+ LOG_UNCORK(WARNING, "last part %d\n", 5);
+ std::stringstream expected;
+ expected << "implementations_test.cc: ";
+ expected << (begin_line + 1);
+ expected << "-";
+ expected << (end_line + 1);
+ expected << ": ";
+ expected << __func__;
+ expected << ": first part second part (=19) third part last part 5\n";
+ EXPECT_TRUE(WasLogged(WARNING, expected.str()));
+}
+
+TEST_F(LoggingDeathTest, Fatal) {
+ ASSERT_EXIT(LOG(FATAL, "this should crash it\n"),
+ ::testing::KilledBySignal(SIGABRT),
+ "this should crash it");
+}
+
+TEST_F(LoggingDeathTest, PCHECK) {
+ EXPECT_DEATH(PCHECK(fprintf(stdin, "nope")),
+ ".*fprintf\\(stdin, \"nope\"\\).*failed.*");
+}
+
+TEST_F(LoggingTest, PCHECK) {
+ EXPECT_EQ(7, PCHECK(printf("abc123\n")));
+}
+
+TEST_F(LoggingTest, PrintfDirectives) {
+ LOG(INFO, "test log %%1 %%d\n");
+ EXPECT_TRUE(WasLogged(INFO, "test log %1 %d\n"));
+ LOG_DYNAMIC(WARNING, "test log %%2 %%f\n");
+ EXPECT_TRUE(WasLogged(WARNING, "test log %2 %f\n"));
+ LOG_CORK("log 3 part %%1 %%d ");
+ LOG_UNCORK(DEBUG, "log 3 part %%2 %%f\n");
+ EXPECT_TRUE(WasLogged(DEBUG, "log 3 part %1 %d log 3 part %2 %f\n"));
+}
+
+TEST_F(LoggingTest, Timing) {
+ // For writing only.
+ //static const long kTimingCycles = 5000000;
+ static const long kTimingCycles = 5000;
+
+ monotonic_clock::time_point start = monotonic_clock::now();
+ for (long i = 0; i < kTimingCycles; ++i) {
+ LOG(INFO, "a\n");
+ }
+ monotonic_clock::time_point end = monotonic_clock::now();
+ auto diff = end - start;
+ printf("short message took %" PRId64 " nsec for %ld\n",
+ chrono::duration_cast<chrono::nanoseconds>(diff).count(),
+ kTimingCycles);
+
+ start = monotonic_clock::now();
+ for (long i = 0; i < kTimingCycles; ++i) {
+ LOG(INFO, "something longer than just \"a\" to log to test timing\n");
+ }
+ end = monotonic_clock::now();
+ diff = end - start;
+ printf("long message took %" PRId64 " nsec for %ld\n",
+ chrono::duration_cast<chrono::nanoseconds>(diff).count(),
+ kTimingCycles);
+}
+
+TEST(LoggingPrintFormatTest, Time) {
+ char buffer[1024];
+
+ // Easy ones.
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(2, 0)));
+ EXPECT_EQ("0000000002.000000s", ::std::string(buffer));
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(2, 1)));
+ EXPECT_EQ("0000000002.000000s", ::std::string(buffer));
+
+ // This one should be exact.
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(2, 1000)));
+ EXPECT_EQ("0000000002.000001s", ::std::string(buffer));
+
+ // Make sure rounding works correctly.
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(2, 999)));
+ EXPECT_EQ("0000000002.000000s", ::std::string(buffer));
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(2, 500)));
+ EXPECT_EQ("0000000002.000000s", ::std::string(buffer));
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(2, 499)));
+ EXPECT_EQ("0000000002.000000s", ::std::string(buffer));
+
+ // This used to result in "0000000001.099500s".
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(1, 995000000)));
+ EXPECT_EQ("0000000001.995000s", ::std::string(buffer));
+
+ // This used to result in "0000000001.099500s".
+ ASSERT_EQ(18, snprintf(buffer, sizeof(buffer), AOS_TIME_FORMAT,
+ AOS_TIME_ARGS(1, 999999999)));
+ EXPECT_EQ("0000000001.999999s", ::std::string(buffer));
+}
+
+TEST(LoggingPrintFormatTest, Base) {
+ char buffer[1024];
+
+ static const ::std::string kExpected1 =
+ "name(971)(01678): ERROR at 0000000001.995000s: ";
+ ASSERT_GT(sizeof(buffer), kExpected1.size());
+ ASSERT_EQ(
+ kExpected1.size(),
+ static_cast<size_t>(snprintf(
+ buffer, sizeof(buffer), AOS_LOGGING_BASE_FORMAT,
+ AOS_LOGGING_BASE_ARGS(4, "name", 971, 1678, ERROR, 1, 995000000))));
+ EXPECT_EQ(kExpected1, ::std::string(buffer));
+}
+
+} // namespace testing
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/interface.cc b/aos/logging/interface.cc
new file mode 100644
index 0000000..fb3b60e
--- /dev/null
+++ b/aos/logging/interface.cc
@@ -0,0 +1,133 @@
+#include "aos/logging/interface.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <type_traits>
+#include <functional>
+
+#include "aos/die.h"
+#include "aos/logging/context.h"
+
+namespace aos {
+namespace logging {
+namespace internal {
+
+size_t ExecuteFormat(char *output, size_t output_size, const char *format,
+ va_list ap) {
+ static const char *const continued = "...\n";
+ const size_t size = output_size - strlen(continued);
+ const int ret = vsnprintf(output, size, format, ap);
+ typedef ::std::common_type<typeof(ret), typeof(size)>::type RetType;
+ if (ret < 0) {
+ PLOG(FATAL, "vsnprintf(%p, %zd, %s, args) failed",
+ output, size, format);
+ } else if (static_cast<RetType>(ret) >= static_cast<RetType>(size)) {
+ // Overwrite the '\0' at the end of the existing data and
+ // copy in the one on the end of continued.
+ memcpy(&output[size - 1], continued, strlen(continued) + 1);
+ }
+ return ::std::min<RetType>(ret, size);
+}
+
+void RunWithCurrentImplementation(
+ int levels, ::std::function<void(LogImplementation *)> function) {
+ Context *context = Context::Get();
+
+ LogImplementation *const top_implementation = context->implementation;
+ LogImplementation *new_implementation = top_implementation;
+ LogImplementation *implementation = NULL;
+ for (int i = 0; i < levels; ++i) {
+ implementation = new_implementation;
+ if (new_implementation == NULL) {
+ Die("no logging implementation to use\n");
+ }
+ new_implementation = new_implementation->next();
+ }
+ context->implementation = new_implementation;
+ function(implementation);
+ context->implementation = top_implementation;
+}
+
+} // namespace internal
+
+using internal::Context;
+
+void LogImplementation::DoVLog(log_level level, const char *format, va_list ap,
+ int levels) {
+ auto log_impl = [&](LogImplementation *implementation) {
+ va_list ap1;
+ va_copy(ap1, ap);
+ implementation->DoLog(level, format, ap1);
+ va_end(ap1);
+
+ if (level == FATAL) {
+ VDie(format, ap);
+ }
+ };
+ internal::RunWithCurrentImplementation(levels, ::std::ref(log_impl));
+}
+
+void VLog(log_level level, const char *format, va_list ap) {
+ LogImplementation::DoVLog(level, format, ap, 1);
+}
+
+void VCork(int line, const char *function, const char *format, va_list ap) {
+ Context *context = Context::Get();
+
+ const size_t message_length = strlen(context->cork_data.message);
+ if (line > context->cork_data.line_max) context->cork_data.line_max = line;
+ if (line < context->cork_data.line_min) context->cork_data.line_min = line;
+
+ if (context->cork_data.function == NULL) {
+ context->cork_data.function = function;
+ } else {
+ if (strcmp(context->cork_data.function, function) != 0) {
+ LOG(FATAL, "started corking data in function %s but then moved to %s\n",
+ context->cork_data.function, function);
+ }
+ }
+
+ internal::ExecuteFormat(context->cork_data.message + message_length,
+ sizeof(context->cork_data.message) - message_length,
+ format, ap);
+}
+
+void VUnCork(int line, const char *function, log_level level, const char *file,
+ const char *format, va_list ap) {
+ Context *context = Context::Get();
+
+ VCork(line, function, format, ap);
+
+ log_do(level, "%s: %d-%d: %s: %s", file,
+ context->cork_data.line_min, context->cork_data.line_max, function,
+ context->cork_data.message);
+
+ context->cork_data.Reset();
+}
+
+} // namespace logging
+} // namespace aos
+
+void log_do(log_level level, const char *format, ...) {
+ va_list ap;
+ va_start(ap, format);
+ aos::logging::VLog(level, format, ap);
+ va_end(ap);
+}
+
+void log_cork(int line, const char *function, const char *format, ...) {
+ va_list ap;
+ va_start(ap, format);
+ aos::logging::VCork(line, function, format, ap);
+ va_end(ap);
+}
+
+void log_uncork(int line, const char *function, log_level level,
+ const char *file, const char *format, ...) {
+ va_list ap;
+ va_start(ap, format);
+ aos::logging::VUnCork(line, function, level, file, format, ap);
+ va_end(ap);
+}
diff --git a/aos/logging/interface.h b/aos/logging/interface.h
new file mode 100644
index 0000000..fafc9e2
--- /dev/null
+++ b/aos/logging/interface.h
@@ -0,0 +1,128 @@
+#ifndef AOS_LOGGING_INTERFACE_H_
+#define AOS_LOGGING_INTERFACE_H_
+
+#include <stdarg.h>
+
+#include <string>
+#include <functional>
+
+#include "aos/logging/logging.h"
+#include "aos/macros.h"
+
+// This file has the non-C-compatible parts of the logging client interface.
+
+namespace aos {
+
+struct MessageType;
+
+namespace logging {
+namespace internal {
+
+// Defined in queue_logging.cc.
+void DoLogStruct(log_level level, const ::std::string &message, size_t size,
+ const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize, int levels);
+
+// Defined in matrix_logging.cc.
+void DoLogMatrix(log_level level, const ::std::string &message,
+ uint32_t type_id, int rows, int cols, const void *data,
+ int levels);
+
+} // namespace internal
+
+// Takes a message and logs it. It will set everything up and then call DoLog
+// for the current LogImplementation.
+void VLog(log_level level, const char *format, va_list ap)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 2, 0)));
+// Adds to the saved up message.
+void VCork(int line, const char *function, const char *format, va_list ap)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)));
+// Actually logs the saved up message.
+void VUnCork(int line, const char *function, log_level level, const char *file,
+ const char *format, va_list ap)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 5, 0)));
+
+// Represents a system that can actually take log messages and do something
+// useful with them.
+// All of the code (transitively too!) in the DoLog here can make
+// normal LOG and LOG_DYNAMIC calls but can NOT call LOG_CORK/LOG_UNCORK. These
+// calls will not result in DoLog recursing. However, implementations must be
+// safe to call from multiple threads/tasks at the same time. Also, any other
+// overriden methods may end up logging through a given implementation's DoLog.
+class LogImplementation {
+ public:
+ LogImplementation() : next_(NULL) {}
+
+ // The one that this one's implementation logs to.
+ // NULL means that there is no next one.
+ LogImplementation *next() { return next_; }
+ // Virtual in case a subclass wants to perform checks. There will be a valid
+ // logger other than this one available while this is called.
+ virtual void set_next(LogImplementation *next) { next_ = next; }
+
+ protected:
+ // Actually logs the given message. Implementations should somehow create a
+ // LogMessage and then call internal::FillInMessage.
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)))
+ virtual void DoLog(log_level level, const char *format, va_list ap) = 0;
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 4)))
+ void DoLogVariadic(log_level level, const char *format, ...) {
+ va_list ap;
+ va_start(ap, format);
+ DoLog(level, format, ap);
+ va_end(ap);
+ }
+
+ // Logs the contents of an auto-generated structure.
+ // size and type are the result of calling Size() and Type() on the type of
+ // the message.
+ // serialize will call Serialize on the message.
+ virtual void LogStruct(log_level level, const ::std::string &message,
+ size_t size, const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize) = 0;
+ // Similiar to LogStruct, except for matrixes.
+ // type_id is the type of the elements of the matrix.
+ // data points to rows*cols*type_id.Size() bytes of data in row-major order.
+ virtual void LogMatrix(log_level level, const ::std::string &message,
+ uint32_t type_id, int rows, int cols,
+ const void *data) = 0;
+
+ private:
+ // These functions call similar methods on the "current" LogImplementation or
+ // Die if they can't find one.
+ // levels is how many LogImplementations to not use off the stack.
+ static void DoVLog(log_level, const char *format, va_list ap, int levels)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 2, 0)));
+
+ friend void VLog(log_level, const char *, va_list);
+ friend void internal::DoLogStruct(
+ log_level level, const ::std::string &message, size_t size,
+ const MessageType *type, const ::std::function<size_t(char *)> &serialize,
+ int levels);
+ friend void internal::DoLogMatrix(log_level level,
+ const ::std::string &message,
+ uint32_t type_id, int rows, int cols,
+ const void *data, int levels);
+
+ LogImplementation *next_;
+};
+
+namespace internal {
+
+// Prints format (with ap) into output and correctly deals with the result
+// being too long etc.
+size_t ExecuteFormat(char *output, size_t output_size, const char *format,
+ va_list ap)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 0)));
+
+// Runs the given function with the current LogImplementation (handles switching
+// it out while running function etc).
+// levels is how many LogImplementations to not use off the stack.
+void RunWithCurrentImplementation(
+ int levels, ::std::function<void(LogImplementation *)> function);
+
+} // namespace internal
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_INTERFACE_H_
diff --git a/aos/logging/log_displayer.cc b/aos/logging/log_displayer.cc
new file mode 100644
index 0000000..1d40de1
--- /dev/null
+++ b/aos/logging/log_displayer.cc
@@ -0,0 +1,411 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <inttypes.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "aos/linux_code/configuration.h"
+#include "aos/logging/binary_log_file.h"
+#include "aos/queue_types.h"
+#include "aos/logging/logging.h"
+#include "aos/logging/implementations.h"
+#include "aos/logging/printf_formats.h"
+#include "aos/util/string_to_num.h"
+
+using ::aos::logging::linux_code::LogFileMessageHeader;
+
+namespace {
+
+const char *kArgsHelp = "[OPTION]... [FILE]\n"
+ "Display log file FILE (created by BinaryLogReader) to stdout.\n"
+ "FILE is \"aos_log-current\" by default.\n"
+ "FILE can also be \"-\" to read from standard input.\n"
+ "\n"
+ " -n, --name-prefix NAME "
+ "only display entries from processes which share NAME as a prefix\n"
+ " -N, --name NAME only display entries from processes named NAME\n"
+ " -l, --level LEVEL "
+ "only display log entries at least as important as LEVEL\n"
+ " -p, --pid PID only display log entries from process PID\n"
+ " -f, --follow "
+ "wait when the end of the file is reached (implies --end)\n"
+ " -t, --terminate stop when the end of file is reached (default)\n"
+ " -b, --beginning start at the beginning of the file (default)\n"
+ " -e, --end start at the end of the file\n"
+ " -s, --skip NUMBER skip NUMBER matching logs\n"
+ " -m, --max NUMBER only display up to NUMBER logs\n"
+ " // -o, --format FORMAT use FORMAT to display log entries\n"
+ " -h, --help display this help and exit\n"
+ "\n"
+ "LEVEL must be DEBUG, INFO, WARNING, ERROR, or FATAL.\n"
+ " It defaults to INFO.\n"
+ "\n"
+ "TODO(brians) implement the commented out ones.\n";
+
+const char *kExampleUsages = "To view logs from the shooter:\n"
+ "\t`log_displayer -n shooter`\n"
+ "To view debug logs from the shooter:\n"
+ "\t`log_displayer -n shooter -l DEBUG`\n"
+ "To view what the shooter is logging in realtime:\n"
+ "\t`log_displayer -f -n shooter`\n"
+ "To view shooter logs from an old log file:\n"
+ "\t`log_displayer aos_log-<number> -n shooter`\n"
+ "To view the statuses of the shooter hall effects in realtime:\n"
+ "\t`log_displayer -f -n shooter -l DEBUG | grep .Position`\n";
+
+void PrintHelpAndExit() {
+ fprintf(stderr, "Usage: %s %s", program_invocation_name, kArgsHelp);
+ fprintf(stderr, "\nExample usages:\n\n%s", kExampleUsages);
+
+ // Get the possible executables from start_list.txt.
+ FILE *start_list = fopen("start_list.txt", "r");
+ if (!start_list) {
+ ::std::string path(::aos::configuration::GetRootDirectory());
+ path += "/start_list.txt";
+ start_list = fopen(path.c_str(), "r");
+ if (!start_list) {
+ printf("\nCannot open start_list.txt. This means that the\n"
+ "possible arguments for the -n option cannot be shown. log_displayer\n"
+ "looks for start_list.txt in the current working directory and in\n"
+ "%s.\n\n", ::aos::configuration::GetRootDirectory());
+ PLOG(FATAL, "Unable to open start_list.txt");
+ }
+ }
+
+ // Get file size.
+ if (fseek(start_list, 0, SEEK_END)) {
+ PLOG(FATAL, "fseek() failed while reading start_list.txt");
+ }
+ int size = ftell(start_list);
+ if (size < 0) {
+ PLOG(FATAL, "ftell() failed while reading start_list.txt");
+ }
+ rewind(start_list);
+
+ ::std::unique_ptr<char[]> contents(new char[size + 1]);
+ if (contents == NULL) {
+ LOG(FATAL, "malloc() failed while reading start_list.txt.\n");
+ }
+ size_t bytes_read = fread(contents.get(), 1, size, start_list);
+ if (bytes_read < static_cast<size_t>(size)) {
+ LOG(FATAL, "Read %zu bytes from start_list.txt, expected %d.\n",
+ bytes_read, size);
+ }
+
+ // printf doesn't like strings without the \0.
+ contents[size] = '\0';
+ fprintf(stderr, "\nPossible arguments for the -n option:\n%s", contents.get());
+
+ if (fclose(start_list)) {
+ LOG(FATAL, "fclose() failed.\n");
+ }
+
+ exit(EXIT_SUCCESS);
+}
+
+} // namespace
+
+int main(int argc, char **argv) {
+ const char *filter_name = nullptr, *filter_exact_name = nullptr;
+ size_t filter_length = 0;
+ log_level filter_level = INFO;
+ bool follow = false;
+ // Whether we need to skip everything until we get to the end of the file.
+ bool skip_to_end = false;
+ const char *filename = "aos_log-current";
+ int display_max = 0;
+ int32_t source_pid = -1;
+
+ ::aos::logging::Init();
+ ::aos::logging::AddImplementation(
+ new ::aos::logging::StreamLogImplementation(stdout));
+
+ while (true) {
+ static struct option long_options[] = {
+ {"name-prefix", required_argument, NULL, 'n'},
+ {"name", required_argument, NULL, 'N'},
+ {"level", required_argument, NULL, 'l'},
+ {"pid", required_argument, NULL, 'p'},
+
+ {"follow", no_argument, NULL, 'f'},
+ {"terminate", no_argument, NULL, 't'},
+ {"beginning", no_argument, NULL, 'b'},
+ {"end", no_argument, NULL, 'e'},
+ {"skip", required_argument, NULL, 's'},
+ {"max", required_argument, NULL, 'm'},
+
+ {"format", required_argument, NULL, 'o'},
+
+ {"help", no_argument, NULL, 'h'},
+ {0, 0, 0, 0}
+ };
+ int option_index = 0;
+
+ const int c = getopt_long(argc, argv, "N:n:l:p:fts:m:o:h",
+ long_options, &option_index);
+ if (c == -1) { // if we're at the end
+ break;
+ }
+ switch (c) {
+ case 0:
+ fputs("LogDisplayer: got a 0 option but didn't set up any\n", stderr);
+ abort();
+ case 'n':
+ filter_name = optarg;
+ filter_exact_name = nullptr;
+ filter_length = strlen(filter_name);
+ break;
+ case 'N':
+ filter_exact_name = optarg;
+ filter_name = nullptr;
+ filter_length = strlen(filter_name);
+ break;
+ case 'l':
+ filter_level = ::aos::logging::str_log(optarg);
+ if (filter_level == LOG_UNKNOWN) {
+ fprintf(stderr, "LogDisplayer: unknown log level '%s'\n", optarg);
+ exit(EXIT_FAILURE);
+ }
+ break;
+ case 'p':
+ if (!::aos::util::StringToNumber(::std::string(optarg), &source_pid)) {
+ fprintf(stderr, "ERROR: -p expects a number, not '%s'.\n", optarg);
+ exit(EXIT_FAILURE);
+ }
+ if (source_pid < 0) {
+ fprintf(stderr, "LogDisplayer: invalid pid '%s'\n", optarg);
+ exit(EXIT_FAILURE);
+ }
+ break;
+ case 'f':
+ follow = true;
+ skip_to_end = true;
+ break;
+ case 't':
+ follow = false;
+ break;
+ case 'b':
+ skip_to_end = false;
+ break;
+ case 'e':
+ skip_to_end = true;
+ break;
+ case 'm':
+ if (!::aos::util::StringToNumber(::std::string(optarg), &display_max)) {
+ fprintf(stderr, "ERROR: -m expects a number, not '%s'.\n", optarg);
+ exit(EXIT_FAILURE);
+ }
+ if (display_max <= 0) {
+ fprintf(stderr, "LogDisplayer: invalid max log number '%s'\n",
+ optarg);
+ exit(EXIT_FAILURE);
+ }
+ break;
+ case 'o':
+ abort();
+ break;
+ case 'h':
+ PrintHelpAndExit();
+ break;
+ case '?':
+ break;
+ default:
+ fprintf(stderr, "LogDisplayer: in a bad spot (%s: %d)\n",
+ __FILE__, __LINE__);
+ abort();
+ }
+ }
+
+ if (optind < argc) {
+ // We got a filename.
+ filename = argv[optind++];
+ }
+ if (optind < argc) {
+ fputs("non-option ARGV-elements: ", stderr);
+ while (optind < argc) {
+ fprintf(stderr, "%s\n", argv[optind++]);
+ }
+ }
+
+ int fd;
+ if (strcmp(filename, "-") == 0) {
+ if (skip_to_end) {
+ fputs("Can't skip to end of stdin!\n", stderr);
+ return EXIT_FAILURE;
+ }
+ fd = STDIN_FILENO;
+ } else {
+ fd = open(filename, O_RDONLY);
+ }
+
+ fprintf(stderr, "displaying down to level %s from file '%s'\n",
+ ::aos::logging::log_str(filter_level), filename);
+
+ if (fd == -1) {
+ PLOG(FATAL, "couldn't open file '%s' for reading", filename);
+ }
+ ::aos::logging::linux_code::LogFileReader reader(fd);
+
+ if (skip_to_end) {
+ fputs("skipping old logs...\n", stderr);
+ reader.SkipToLastSeekablePage();
+ }
+
+ const LogFileMessageHeader *msg;
+ int displayed = 0;
+ do {
+ msg = reader.ReadNextMessage(follow);
+ if (msg == NULL) {
+ fputs("reached end of file\n", stderr);
+ return 0;
+ }
+
+ if (msg->type == LogFileMessageHeader::MessageType::kStructType) {
+ size_t bytes = msg->message_size;
+ ::aos::MessageType *type = ::aos::MessageType::Deserialize(
+ reinterpret_cast<const char *>(msg + 1), &bytes);
+ if (type == nullptr) {
+ LOG(INFO, "Trying old version of type decoding.\n");
+ bytes = msg->message_size;
+ type = ::aos::MessageType::Deserialize(
+ reinterpret_cast<const char *>(msg + 1), &bytes, false);
+ }
+
+ if (type == nullptr) {
+ LOG(WARNING, "Error deserializing MessageType of size %" PRIx32
+ " starting at %zx.\n",
+ msg->message_size, reader.file_offset(msg + 1));
+ } else {
+ ::aos::type_cache::Add(*type);
+ }
+ continue;
+ }
+
+ if (source_pid >= 0 && msg->source != source_pid) {
+ // Message is from the wrong process.
+ continue;
+ }
+
+ if (skip_to_end) {
+ if (reader.IsLastPage()) {
+ fputs("done skipping old logs\n", stderr);
+ skip_to_end = false;
+ } else {
+ continue;
+ }
+ }
+
+ if (::aos::logging::log_gt_important(filter_level, msg->level)) continue;
+
+ const char *position =
+ reinterpret_cast<const char *>(msg + 1);
+
+ if (filter_name != nullptr) {
+ const size_t compare_length =
+ ::std::min<size_t>(filter_length, msg->name_size);
+ if (memcmp(filter_name, position, compare_length) != 0) {
+ continue;
+ }
+ if (compare_length < msg->name_size) {
+ if (position[compare_length] != '.') continue;
+ }
+ }
+
+ if (filter_exact_name != nullptr) {
+ if (filter_length != msg->name_size) continue;
+ if (memcmp(filter_exact_name, position, filter_length) != 0) {
+ continue;
+ }
+ }
+
+ if (display_max && displayed++ >= display_max) {
+ fputs("Not displaying the rest of the messages.\n", stderr);
+ return 0;
+ }
+
+ position += msg->name_size;
+
+#define BASE_ARGS \
+ AOS_LOGGING_BASE_ARGS( \
+ msg->name_size, reinterpret_cast<const char *>(msg + 1), msg->source, \
+ msg->sequence, msg->level, msg->time_sec, msg->time_nsec)
+ switch (msg->type) {
+ case LogFileMessageHeader::MessageType::kString:
+ fprintf(stdout, AOS_LOGGING_BASE_FORMAT "%.*s", BASE_ARGS,
+ static_cast<int>(msg->message_size), position);
+ break;
+ case LogFileMessageHeader::MessageType::kStruct: {
+ uint32_t type_id;
+ memcpy(&type_id, position, sizeof(type_id));
+ position += sizeof(type_id);
+
+ uint32_t string_length;
+ memcpy(&string_length, position, sizeof(string_length));
+ position += sizeof(string_length);
+
+ char buffer[4096];
+ size_t output_length = sizeof(buffer);
+ size_t input_length =
+ msg->message_size -
+ (sizeof(type_id) + sizeof(uint32_t) + string_length);
+ if (!PrintMessage(buffer, &output_length, position + string_length,
+ &input_length, ::aos::type_cache::Get(type_id))) {
+ LOG(FATAL, "printing message (%.*s) of type %s into %zu-byte buffer "
+ "failed\n",
+ static_cast<int>(string_length), position,
+ ::aos::type_cache::Get(type_id).name.c_str(), sizeof(buffer));
+ }
+ if (input_length > 0) {
+ LOG(WARNING, "%zu extra bytes on message of type %s\n",
+ input_length, ::aos::type_cache::Get(type_id).name.c_str());
+ }
+ fprintf(stdout, AOS_LOGGING_BASE_FORMAT "%.*s: %.*s\n", BASE_ARGS,
+ static_cast<int>(string_length), position,
+ static_cast<int>(sizeof(buffer) - output_length), buffer);
+ } break;
+ case LogFileMessageHeader::MessageType::kMatrix: {
+ uint32_t type;
+ memcpy(&type, position, sizeof(type));
+ position += sizeof(type);
+
+ uint32_t string_length;
+ memcpy(&string_length, position, sizeof(string_length));
+ position += sizeof(string_length);
+
+ uint16_t rows;
+ memcpy(&rows, position, sizeof(rows));
+ position += sizeof(rows);
+ uint16_t cols;
+ memcpy(&cols, position, sizeof(cols));
+ position += sizeof(cols);
+
+ const size_t matrix_bytes =
+ msg->message_size -
+ (sizeof(type) + sizeof(uint32_t) + sizeof(uint16_t) +
+ sizeof(uint16_t) + string_length);
+ CHECK_EQ(matrix_bytes, ::aos::MessageType::Sizeof(type) * rows * cols);
+ char buffer[4096];
+ size_t output_length = sizeof(buffer);
+ if (!::aos::PrintMatrix(buffer, &output_length,
+ position + string_length, type, rows, cols)) {
+ LOG(FATAL, "printing %dx%d matrix of type %" PRIu32 " failed\n", rows,
+ cols, type);
+ }
+ fprintf(stdout, AOS_LOGGING_BASE_FORMAT "%.*s: %.*s\n", BASE_ARGS,
+ static_cast<int>(string_length), position,
+ static_cast<int>(sizeof(buffer) - output_length), buffer);
+ } break;
+ case LogFileMessageHeader::MessageType::kStructType:
+ LOG(FATAL, "shouldn't get here\n");
+ break;
+ }
+#undef BASE_ARGS
+ } while (msg != NULL);
+}
diff --git a/aos/logging/log_streamer.cc b/aos/logging/log_streamer.cc
new file mode 100644
index 0000000..757ba2e
--- /dev/null
+++ b/aos/logging/log_streamer.cc
@@ -0,0 +1,63 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+#include <chrono>
+#include <string>
+
+#include "aos/logging/implementations.h"
+#include "aos/logging/logging.h"
+#include "aos/time/time.h"
+#include "aos/linux_code/init.h"
+#include "aos/linux_code/ipc_lib/queue.h"
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+namespace {
+
+namespace chrono = ::std::chrono;
+
+int LogStreamerMain() {
+ InitNRT();
+
+ RawQueue *queue = GetLoggingQueue();
+
+ const monotonic_clock::time_point now = monotonic_clock::now();
+ chrono::seconds sec =
+ chrono::duration_cast<chrono::seconds>(now.time_since_epoch());
+ chrono::nanoseconds nsec =
+ chrono::duration_cast<chrono::nanoseconds>(now.time_since_epoch() - sec);
+ printf("starting at %" PRId32 "s%" PRId32 "ns-----------------------------\n",
+ static_cast<int32_t>(sec.count()), static_cast<int32_t>(nsec.count()));
+
+ while (true) {
+ const LogMessage *const msg = static_cast<const LogMessage *>(
+ queue->ReadMessage(RawQueue::kNonBlock));
+ if (msg == NULL) {
+ ::std::this_thread::sleep_for(::std::chrono::milliseconds(100));
+ } else {
+ internal::PrintMessage(stdout, *msg);
+
+ queue->FreeMessage(msg);
+ }
+ }
+
+ Cleanup();
+ return 0;
+}
+
+} // namespace
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
+
+int main() {
+ return ::aos::logging::linux_code::LogStreamerMain();
+}
diff --git a/aos/logging/logging.h b/aos/logging/logging.h
new file mode 100644
index 0000000..6682098
--- /dev/null
+++ b/aos/logging/logging.h
@@ -0,0 +1,258 @@
+#ifndef AOS_LOGGING_LOGGING_H_
+#define AOS_LOGGING_LOGGING_H_
+
+// This file contains the logging client interface. It works with both C and C++
+// code.
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "aos/macros.h"
+#include "aos/libc/aos_strerror.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef uint8_t log_level;
+
+#define DECL_LEVELS \
+DECL_LEVEL(DEBUG, 0); /* stuff that gets printed out every cycle */ \
+DECL_LEVEL(INFO, 1); /* things like PosEdge/NegEdge */ \
+/* things that might still work if they happen occasionally */ \
+DECL_LEVEL(WARNING, 2); \
+/*-1 so that vxworks macro of same name will have same effect if used*/ \
+DECL_LEVEL(ERROR, -1); /* errors */ \
+/* serious errors. the logging code will terminate the process/task */ \
+DECL_LEVEL(FATAL, 4); \
+DECL_LEVEL(LOG_UNKNOWN, 5); /* unknown logging level */
+#define DECL_LEVEL(name, value) static const log_level name = value;
+DECL_LEVELS;
+#undef DECL_LEVEL
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Actually implements the basic logging call.
+// Does not check that level is valid.
+void log_do(log_level level, const char *format, ...)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 2, 3)));
+
+void log_cork(int line, const char *function, const char *format, ...)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 3, 4)));
+// Implements the uncork logging call.
+void log_uncork(int line, const char *function, log_level level,
+ const char *file, const char *format, ...)
+ __attribute__((format(GOOD_PRINTF_FORMAT_TYPE, 5, 6)));
+
+#ifdef __cplusplus
+}
+#endif
+
+// A magical static const char[] or string literal that communicates the name
+// of the enclosing function.
+// It's currently using __PRETTY_FUNCTION__ because both GCC and Clang support
+// that and it gives nicer results in C++ than the standard __func__ (which
+// would also work).
+//#define LOG_CURRENT_FUNCTION __PRETTY_FUNCTION__
+#define LOG_CURRENT_FUNCTION __func__
+
+#define LOG_SOURCENAME __FILE__
+
+// The basic logging call.
+#define LOG(level, format, args...) \
+ do { \
+ log_do(level, LOG_SOURCENAME ": " STRINGIFY(__LINE__) ": %s: " format, \
+ LOG_CURRENT_FUNCTION, ##args); \
+ /* so that GCC knows that it won't return */ \
+ if (level == FATAL) { \
+ fprintf(stderr, "log_do(FATAL) fell through!!!!!\n"); \
+ printf("see stderr\n"); \
+ abort(); \
+ } \
+ } while (0)
+
+// Same as LOG except appends " due to %d (%s)\n" (formatted with errno and
+// aos_strerror(errno)) to the message.
+#define PLOG(level, format, args...) PELOG(level, errno, format, ##args)
+
+// Like PLOG except allows specifying an error other than errno.
+#define PELOG(level, error_in, format, args...) \
+ do { \
+ const int error = error_in; \
+ LOG(level, format " due to %d (%s)\n", ##args, error, \
+ aos_strerror(error)); \
+ } while (0);
+
+// Allows format to not be a string constant.
+#define LOG_DYNAMIC(level, format, args...) \
+ do { \
+ static char log_buf[LOG_MESSAGE_LEN]; \
+ int ret = snprintf(log_buf, sizeof(log_buf), format, ##args); \
+ if (ret < 0 || (uintmax_t)ret >= LOG_MESSAGE_LEN) { \
+ LOG(ERROR, "next message was too long so not subbing in args\n"); \
+ LOG(level, "%s", format); \
+ } else { \
+ LOG(level, "%s", log_buf); \
+ } \
+ } while (0)
+
+// Allows "bottling up" multiple log fragments which can then all be logged in
+// one message with LOG_UNCORK.
+// Calls from a given thread/task will be grouped together.
+#define LOG_CORK(format, args...) \
+ do { \
+ log_cork(__LINE__, LOG_CURRENT_FUNCTION, format, ##args); \
+ } while (0)
+// Actually logs all of the saved up log fragments (including format and args on
+// the end).
+#define LOG_UNCORK(level, format, args...) \
+ do { \
+ log_uncork(__LINE__, LOG_CURRENT_FUNCTION, level, LOG_SOURCENAME, format, \
+ ##args); \
+ } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+
+namespace aos {
+
+// CHECK* macros, similar to glog
+// (<http://google-glog.googlecode.com/svn/trunk/doc/glog.html>)'s, except they
+// don't support streaming in extra text. Some of the implementation is borrowed
+// from there too.
+// They all LOG(FATAL) with a helpful message when the check fails.
+// Portions copyright (c) 1999, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// CHECK dies with a fatal error if condition is not true. It is *not*
+// controlled by NDEBUG, so the check will be executed regardless of
+// compilation mode. Therefore, it is safe to do things like:
+// CHECK(fp->Write(x) == 4)
+#define CHECK(condition) \
+ if (__builtin_expect(!(condition), 0)) { \
+ LOG(FATAL, "CHECK(%s) failed\n", #condition); \
+ }
+
+// Helper functions for CHECK_OP macro.
+// The (int, int) specialization works around the issue that the compiler
+// will not instantiate the template version of the function on values of
+// unnamed enum type.
+#define DEFINE_CHECK_OP_IMPL(name, op) \
+ template <typename T1, typename T2> \
+ inline void LogImpl##name(const T1 &v1, const T2 &v2, \
+ const char *exprtext) { \
+ if (!__builtin_expect(v1 op v2, 1)) { \
+ log_do(FATAL, \
+ LOG_SOURCENAME ": " STRINGIFY(__LINE__) ": CHECK(%s) failed\n", \
+ exprtext); \
+ fprintf(stderr, "log_do(FATAL) fell through!!!!!\n"); \
+ printf("see stderr\n"); \
+ abort(); \
+ } \
+ } \
+ inline void LogImpl##name(int v1, int v2, const char *exprtext) { \
+ ::aos::LogImpl##name<int, int>(v1, v2, exprtext); \
+ }
+
+// We use the full name Check_EQ, Check_NE, etc. in case the file including
+// base/logging.h provides its own #defines for the simpler names EQ, NE, etc.
+// This happens if, for example, those are used as token names in a
+// yacc grammar.
+DEFINE_CHECK_OP_IMPL(Check_EQ, ==) // Compilation error with CHECK_EQ(NULL, x)?
+DEFINE_CHECK_OP_IMPL(Check_NE, !=) // Use CHECK(x == NULL) instead.
+DEFINE_CHECK_OP_IMPL(Check_LE, <=)
+DEFINE_CHECK_OP_IMPL(Check_LT, < )
+DEFINE_CHECK_OP_IMPL(Check_GE, >=)
+DEFINE_CHECK_OP_IMPL(Check_GT, > )
+
+#define CHECK_OP(name, op, val1, val2) \
+ ::aos::LogImplCheck##name(val1, val2, \
+ STRINGIFY(val1) STRINGIFY(op) STRINGIFY(val2))
+
+#define CHECK_EQ(val1, val2) CHECK_OP(_EQ, ==, val1, val2)
+#define CHECK_NE(val1, val2) CHECK_OP(_NE, !=, val1, val2)
+#define CHECK_LE(val1, val2) CHECK_OP(_LE, <=, val1, val2)
+#define CHECK_LT(val1, val2) CHECK_OP(_LT, < , val1, val2)
+#define CHECK_GE(val1, val2) CHECK_OP(_GE, >=, val1, val2)
+#define CHECK_GT(val1, val2) CHECK_OP(_GT, > , val1, val2)
+
+// A small helper for CHECK_NOTNULL().
+template <typename T>
+inline T* CheckNotNull(const char *value_name, T *t) {
+ if (t == NULL) {
+ LOG(FATAL, "'%s' must not be NULL\n", value_name);
+ }
+ return t;
+}
+
+// Check that the input is non NULL. This very useful in constructor
+// initializer lists.
+#define CHECK_NOTNULL(val) ::aos::CheckNotNull(STRINGIFY(val), val)
+
+inline int CheckSyscall(const char *syscall_string, int value) {
+ if (__builtin_expect(value == -1, false)) {
+ PLOG(FATAL, "%s failed", syscall_string);
+ }
+ return value;
+}
+
+inline void CheckSyscallReturn(const char *syscall_string, int value) {
+ if (__builtin_expect(value != 0, false)) {
+ PELOG(FATAL, value, "%s failed", syscall_string);
+ }
+}
+
+// Check that syscall does not return -1. If it does, PLOG(FATAL)s. This is
+// useful for quickly checking syscalls where it's not very useful to print out
+// the values of any of the arguments. Returns the result otherwise.
+//
+// Example: const int fd = PCHECK(open("/tmp/whatever", O_WRONLY))
+#define PCHECK(syscall) ::aos::CheckSyscall(STRINGIFY(syscall), syscall)
+
+// PELOG(FATAL)s with the result of syscall if it returns anything other than 0.
+// This is useful for quickly checking things like many of the pthreads
+// functions where it's not very useful to print out the values of any of the
+// arguments.
+//
+// Example: PRCHECK(munmap(address, length))
+#define PRCHECK(syscall) ::aos::CheckSyscallReturn(STRINGIFY(syscall), syscall)
+
+} // namespace aos
+
+#endif // __cplusplus
+
+#endif
diff --git a/aos/logging/matrix_logging.cc b/aos/logging/matrix_logging.cc
new file mode 100644
index 0000000..e17c8a6
--- /dev/null
+++ b/aos/logging/matrix_logging.cc
@@ -0,0 +1,39 @@
+#include "aos/logging/matrix_logging.h"
+
+#include "aos/queue_types.h"
+#include "aos/logging/sizes.h"
+
+namespace aos {
+namespace logging {
+namespace internal {
+
+void DoLogMatrix(log_level level, const ::std::string &message,
+ uint32_t type_id, int rows, int cols, const void *data,
+ int levels) {
+ {
+ auto fn = [&](LogImplementation *implementation) {
+ implementation->LogMatrix(level, message, type_id, rows, cols, data);
+ };
+ RunWithCurrentImplementation(levels, ::std::ref(fn));
+ }
+
+ if (level == FATAL) {
+ char serialized[1024];
+ if (static_cast<size_t>(rows * cols * MessageType::Sizeof(type_id)) >
+ sizeof(serialized)) {
+ Die("LOG(FATAL) matrix too big to serialize");
+ }
+ SerializeMatrix(type_id, serialized, data, rows, cols);
+ char printed[LOG_MESSAGE_LEN];
+ size_t printed_bytes = sizeof(printed);
+ if (!PrintMatrix(printed, &printed_bytes, serialized, type_id, rows, cols)) {
+ Die("LOG(FATAL) PrintMatrix call failed");
+ }
+ Die("%.*s: %.*s\n", static_cast<int>(message.size()), message.data(),
+ static_cast<int>(printed_bytes), printed);
+ }
+}
+
+} // namespace internal
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/matrix_logging.h b/aos/logging/matrix_logging.h
new file mode 100644
index 0000000..630e85e
--- /dev/null
+++ b/aos/logging/matrix_logging.h
@@ -0,0 +1,47 @@
+#ifndef AOS_LOGGING_MATRIX_LOGGING_H_
+#define AOS_LOGGING_MATRIX_LOGGING_H_
+
+#include <string>
+#include <functional>
+
+#include "Eigen/Dense"
+
+#include "aos/logging/interface.h"
+#include "aos/die.h"
+#include "aos/queue_primitives.h"
+
+namespace aos {
+namespace logging {
+
+// Logs the contents of a matrix and a constant string.
+// matrix must be an instance of an Eigen matrix (or something similar).
+#define LOG_MATRIX(level, message, matrix) \
+ do { \
+ static const ::std::string kAosLoggingMessage( \
+ LOG_SOURCENAME ": " STRINGIFY(__LINE__) ": " message); \
+ ::aos::logging::DoLogMatrixTemplated(level, kAosLoggingMessage, \
+ (matrix).eval()); \
+ /* so that GCC knows that it won't return */ \
+ if (level == FATAL) { \
+ ::aos::Die("DoLogStruct(FATAL) fell through!!!!!\n"); \
+ } \
+ } while (false)
+
+template <class T>
+void DoLogMatrixTemplated(log_level level, const ::std::string &message,
+ const T &matrix) {
+ if (T::IsRowMajor) {
+ typename T::Scalar data[matrix.rows() * matrix.cols()];
+ ::Eigen::Map<T>(data, matrix.rows(), matrix.cols()) = matrix;
+ internal::DoLogMatrix(level, message, TypeID<typename T::Scalar>::id,
+ matrix.rows(), matrix.cols(), data, 1);
+ } else {
+ internal::DoLogMatrix(level, message, TypeID<typename T::Scalar>::id,
+ matrix.rows(), matrix.cols(), matrix.data(), 1);
+ }
+}
+
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_MATRIX_LOGGING_H_
diff --git a/aos/logging/printf_formats.h b/aos/logging/printf_formats.h
new file mode 100644
index 0000000..a04f1f2
--- /dev/null
+++ b/aos/logging/printf_formats.h
@@ -0,0 +1,28 @@
+#ifndef AOS_LOGGING_PRINTF_FORMATS_H_
+#define AOS_LOGGING_PRINTF_FORMATS_H_
+
+#include "aos/macros.h"
+
+// This file has printf(3) formats and corresponding arguments for printing out
+// times and log messages.
+// They are all split out as macros because there are 2 things that want to
+// print using the same format: log_displayer and PrintMessage in
+// implementations.cc.
+
+#define AOS_TIME_FORMAT \
+ "%010" PRId32 ".%0" STRINGIFY(AOS_TIME_NSECONDS_DIGITS) PRId32 "s"
+#define AOS_TIME_ARGS(sec, nsec) sec, (nsec / AOS_TIME_NSECONDS_DENOMINATOR)
+
+#define AOS_LOGGING_BASE_FORMAT \
+ "%.*s(%" PRId32 ")(%05" PRIu16 "): %-7s at " AOS_TIME_FORMAT ": "
+#define AOS_LOGGING_BASE_ARGS(name_length, name, source, sequence, level, sec, \
+ nsec) \
+ static_cast<int>(name_length), name, source, sequence, \
+ ::aos::logging::log_str(level), AOS_TIME_ARGS(sec, nsec)
+
+// These 2 define how many digits we use to print out the nseconds fields of
+// times. They have to stay matching.
+#define AOS_TIME_NSECONDS_DIGITS 6
+#define AOS_TIME_NSECONDS_DENOMINATOR 1000
+
+#endif // AOS_LOGGING_PRINTF_FORMATS_H_
diff --git a/aos/logging/queue_logging.cc b/aos/logging/queue_logging.cc
new file mode 100644
index 0000000..541ddb1
--- /dev/null
+++ b/aos/logging/queue_logging.cc
@@ -0,0 +1,39 @@
+#include "aos/logging/queue_logging.h"
+
+#include "aos/logging/interface.h"
+#include "aos/logging/sizes.h"
+#include "aos/queue_types.h"
+
+namespace aos {
+namespace logging {
+namespace internal {
+
+void DoLogStruct(log_level level, const ::std::string &message, size_t size,
+ const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize, int levels) {
+ {
+ auto fn = [&](LogImplementation *implementation) {
+ implementation->LogStruct(level, message, size, type, serialize);
+ };
+ RunWithCurrentImplementation(levels, ::std::ref(fn));
+ }
+
+ if (level == FATAL) {
+ char serialized[1024];
+ if (size > sizeof(serialized)) {
+ Die("LOG(FATAL) structure too big to serialize");
+ }
+ size_t used = serialize(serialized);
+ char printed[LOG_MESSAGE_LEN];
+ size_t printed_bytes = sizeof(printed);
+ if (!PrintMessage(printed, &printed_bytes, serialized, &used, *type)) {
+ Die("LOG(FATAL) PrintMessage call failed");
+ }
+ Die("%.*s: %.*s\n", static_cast<int>(message.size()), message.data(),
+ static_cast<int>(printed_bytes), printed);
+ }
+}
+
+} // namespace internal
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/queue_logging.h b/aos/logging/queue_logging.h
new file mode 100644
index 0000000..a390412
--- /dev/null
+++ b/aos/logging/queue_logging.h
@@ -0,0 +1,43 @@
+#ifndef AOS_LOGGING_QUEUE_LOGGING_H_
+#define AOS_LOGGING_QUEUE_LOGGING_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <functional>
+#include <string>
+
+#include "aos/logging/interface.h"
+#include "aos/die.h"
+
+namespace aos {
+namespace logging {
+
+// Logs the contents of a structure (or Queue message) and a constant string.
+// structure must be an instance of one of the generated queue types.
+#define LOG_STRUCT(level, message, structure) \
+ do { \
+ static const ::std::string kAosLoggingMessage( \
+ LOG_SOURCENAME ": " STRINGIFY(__LINE__) ": " message); \
+ ::aos::logging::DoLogStructTemplated(level, kAosLoggingMessage, \
+ structure); \
+ /* so that GCC knows that it won't return */ \
+ if (level == FATAL) { \
+ ::aos::Die("DoLogStruct(FATAL) fell through!!!!!\n"); \
+ } \
+ } while (false)
+
+template <class T>
+void DoLogStructTemplated(log_level level, const ::std::string &message,
+ const T &structure) {
+ auto fn = [&structure](char *buffer)
+ -> size_t { return structure.Serialize(buffer); };
+
+ internal::DoLogStruct(level, message, T::Size(), T::GetType(), ::std::ref(fn),
+ 1);
+}
+
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_QUEUE_LOGGING_H_
diff --git a/aos/logging/replay.cc b/aos/logging/replay.cc
new file mode 100644
index 0000000..c58ee8c
--- /dev/null
+++ b/aos/logging/replay.cc
@@ -0,0 +1,49 @@
+#include "aos/logging/replay.h"
+
+#include <chrono>
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+
+namespace chrono = ::std::chrono;
+
+bool LogReplayer::ProcessMessage() {
+ const LogFileMessageHeader *message = reader_->ReadNextMessage(false);
+ if (message == nullptr) return true;
+ if (message->type != LogFileMessageHeader::MessageType::kStruct) return false;
+
+ const char *position = reinterpret_cast<const char *>(message + 1);
+
+ ::std::string process(position, message->name_size);
+ position += message->name_size;
+
+ uint32_t type_id;
+ memcpy(&type_id, position, sizeof(type_id));
+ position += sizeof(type_id);
+
+ uint32_t message_length;
+ memcpy(&message_length, position, sizeof(message_length));
+ position += sizeof(message_length);
+ ::std::string message_text(position, message_length);
+ position += message_length;
+
+ size_t split_index = message_text.find_first_of(':') + 2;
+ split_index = message_text.find_first_of(':', split_index) + 2;
+ message_text = message_text.substr(split_index);
+
+ auto handler = handlers_.find(Key(process, message_text));
+ if (handler == handlers_.end()) return false;
+
+ handler->second->HandleStruct(
+ monotonic_clock::time_point(chrono::seconds(message->time_sec) +
+ chrono::nanoseconds(message->time_nsec)),
+ type_id, position,
+ message->message_size -
+ (sizeof(type_id) + sizeof(message_length) + message_length));
+ return false;
+}
+
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
diff --git a/aos/logging/replay.h b/aos/logging/replay.h
new file mode 100644
index 0000000..0de207a
--- /dev/null
+++ b/aos/logging/replay.h
@@ -0,0 +1,166 @@
+#ifndef AOS_LOGGING_REPLAY_H_
+#define AOS_LOGGING_REPLAY_H_
+
+#include <unordered_map>
+#include <string>
+#include <functional>
+#include <memory>
+
+#include "aos/logging/binary_log_file.h"
+#include "aos/queue.h"
+#include "aos/logging/logging.h"
+#include "aos/macros.h"
+#include "aos/linux_code/ipc_lib/queue.h"
+#include "aos/queue_types.h"
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+
+// Manages pulling logged queue messages out of log files.
+//
+// Basic usage:
+// - Use the Add* methods to register handlers for various message sources.
+// - Call OpenFile to open a log file.
+// - Call ProcessMessage repeatedly until it returns true.
+//
+// This code could do something to adapt similar-but-not-identical
+// messages to the current versions, but currently it will LOG(FATAL) if any of
+// the messages don't match up exactly.
+class LogReplayer {
+ public:
+ LogReplayer() {}
+
+ // Gets ready to read messages from fd.
+ // Does not take ownership of fd.
+ void OpenFile(int fd) {
+ reader_.reset(new LogFileReader(fd));
+ }
+ // Closes the currently open file.
+ void CloseCurrentFile() { reader_.reset(); }
+ // Returns true if we have a file which is currently open.
+ bool HasCurrentFile() const { return reader_.get() != nullptr; }
+
+ // Processes a single message from the currently open file.
+ // Returns true if there are no more messages in the file.
+ // This will not call any of the handlers if the next message either has no
+ // registered handlers or is not a queue message.
+ bool ProcessMessage();
+
+ // Adds a handler for messages with a certain string from a certain process.
+ // T must be a Message with the same format as the messages generated by
+ // the .q files.
+ // LOG(FATAL)s for duplicate handlers.
+ template <class T>
+ void AddHandler(const ::std::string &process_name,
+ const ::std::string &log_message,
+ ::std::function<void(const T &message)> handler) {
+ CHECK(handlers_.emplace(
+ ::std::piecewise_construct,
+ ::std::forward_as_tuple(process_name, log_message),
+ ::std::forward_as_tuple(::std::unique_ptr<StructHandlerInterface>(
+ new TypedStructHandler<T>(handler)))).second);
+ }
+
+ // Adds a handler which takes messages and places them directly on a queue.
+ // T must be a Message with the same format as the messages generated by
+ // the .q files.
+ template <class T>
+ void AddDirectQueueSender(const ::std::string &process_name,
+ const ::std::string &log_message,
+ const ::aos::Queue<T> &queue) {
+ AddHandler(process_name, log_message,
+ ::std::function<void(const T &)>(
+ QueueDumpStructHandler<T>(queue.name())));
+ }
+
+ private:
+ // A generic handler of struct log messages.
+ class StructHandlerInterface {
+ public:
+ virtual ~StructHandlerInterface() {}
+
+ virtual void HandleStruct(::aos::monotonic_clock::time_point log_time,
+ uint32_t type_id, const void *data,
+ size_t data_size) = 0;
+ };
+
+ // Converts struct log messages to a message type and passes it to an
+ // ::std::function.
+ template <class T>
+ class TypedStructHandler : public StructHandlerInterface {
+ public:
+ TypedStructHandler(::std::function<void(const T &message)> handler)
+ : handler_(handler) {}
+
+ void HandleStruct(::aos::monotonic_clock::time_point log_time,
+ uint32_t type_id, const void *data,
+ size_t data_size) override {
+ CHECK_EQ(type_id, T::GetType()->id);
+ T message;
+ CHECK_EQ(data_size, T::Size());
+ CHECK_EQ(data_size, message.Deserialize(static_cast<const char *>(data)));
+ message.sent_time = log_time;
+ handler_(message);
+ }
+
+ private:
+ const ::std::function<void(T message)> handler_;
+ };
+
+ // A callable class which dumps messages straight to a queue.
+ template <class T>
+ class QueueDumpStructHandler {
+ public:
+ QueueDumpStructHandler(const ::std::string &queue_name)
+ : queue_(RawQueue::Fetch(queue_name.c_str(), sizeof(T), T::kHash,
+ T::kQueueLength)) {}
+
+ void operator()(const T &message) {
+ LOG_STRUCT(DEBUG, "re-sending", message);
+ void *raw_message = queue_->GetMessage();
+ CHECK_NOTNULL(raw_message);
+ memcpy(raw_message, &message, sizeof(message));
+ CHECK(queue_->WriteMessage(raw_message, RawQueue::kOverride));
+ }
+
+ private:
+ ::aos::RawQueue *const queue_;
+ };
+
+ // A key for specifying log messages to give to a certain handler.
+ struct Key {
+ Key(const ::std::string &process_name, const ::std::string &log_message)
+ : process_name(process_name), log_message(log_message) {}
+
+ ::std::string process_name;
+ ::std::string log_message;
+ };
+
+ struct KeyHash {
+ size_t operator()(const Key &key) const {
+ return string_hash(key.process_name) ^
+ (string_hash(key.log_message) << 1);
+ }
+
+ private:
+ const ::std::hash<::std::string> string_hash = ::std::hash<::std::string>();
+ };
+ struct KeyEqual {
+ bool operator()(const Key &a, const Key &b) const {
+ return a.process_name == b.process_name && a.log_message == b.log_message;
+ }
+ };
+
+ ::std::unordered_map<const Key, ::std::unique_ptr<StructHandlerInterface>,
+ KeyHash, KeyEqual> handlers_;
+ ::std::unique_ptr<LogFileReader> reader_;
+
+ DISALLOW_COPY_AND_ASSIGN(LogReplayer);
+};
+
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
+
+#endif // AOS_LOGGING_REPLAY_H_
diff --git a/aos/logging/sizes.h b/aos/logging/sizes.h
new file mode 100644
index 0000000..b624a61
--- /dev/null
+++ b/aos/logging/sizes.h
@@ -0,0 +1,9 @@
+#ifndef AOS_LOGGING_SIZES_H_
+#define AOS_LOGGING_SIZES_H_
+
+// This file exists so C code and context.h can both get at these constants...
+
+#define LOG_MESSAGE_LEN 500
+#define LOG_MESSAGE_NAME_LEN 100
+
+#endif // AOS_LOGGING_SIZES_H_