cleaned up the existing logging code
diff --git a/aos/linux_code/logging/binary_log_file.cc b/aos/linux_code/logging/binary_log_file.cc
new file mode 100644
index 0000000..e1e99b4
--- /dev/null
+++ b/aos/linux_code/logging/binary_log_file.cc
@@ -0,0 +1,123 @@
+#include "aos/linux_code/logging/binary_log_file.h"
+
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+namespace aos {
+namespace logging {
+namespace linux_code {
+namespace {
+
+unsigned long SystemPageSize() {
+ static unsigned long r = sysconf(_SC_PAGESIZE);
+ return r;
+}
+
+} // namespace
+
+LogFileAccessor::LogFileAccessor(int fd, bool writable)
+ : fd_(fd), writable_(writable), offset_(0), current_(0), position_(0) {
+ // Check to make sure that mmap will allow mmaping in chunks of kPageSize.
+ if (SystemPageSize() > kPageSize || (kPageSize % SystemPageSize()) != 0) {
+ LOG(FATAL, "system page size (%lu) not factor of kPageSize (%zd).\n",
+ SystemPageSize(), kPageSize);
+ }
+
+ MapNextPage();
+}
+
+LogFileMessageHeader *LogFileAccessor::GetWritePosition(size_t message_size) {
+ if (position_ + message_size + (kAlignment - (message_size % kAlignment)) +
+ sizeof(mutex) > kPageSize) {
+ char *const temp = current_;
+ MapNextPage();
+ if (futex_set_value(static_cast<mutex *>(static_cast<void *>(
+ &temp[position_])), 2) == -1) {
+ LOG(WARNING,
+ "futex_set_value(%p, 2) failed with %d: %s. readers will hang\n",
+ &temp[position_], errno, strerror(errno));
+ }
+ Unmap(temp);
+ position_ = 0;
+ }
+ LogFileMessageHeader *const r = static_cast<LogFileMessageHeader *>(
+ static_cast<void *>(¤t_[position_]));
+ position_ += message_size;
+ AlignPosition();
+ return r;
+}
+
+const LogFileMessageHeader *LogFileAccessor::ReadNextMessage(bool wait) {
+ LogFileMessageHeader *r;
+ do {
+ r = static_cast<LogFileMessageHeader *>(
+ static_cast<void *>(¤t_[position_]));
+ if (wait) {
+ if (futex_wait(&r->marker) != 0) continue;
+ }
+ if (r->marker == 2) {
+ Unmap(current_);
+ MapNextPage();
+ position_ = 0;
+ r = static_cast<LogFileMessageHeader *>(static_cast<void *>(current_));
+ }
+ } while (wait && r->marker == 0);
+ if (r->marker == 0) {
+ return NULL;
+ }
+ position_ += sizeof(LogFileMessageHeader) + r->name_size + r->message_size;
+ AlignPosition();
+ return r;
+}
+
+void LogFileAccessor::Sync() const {
+ msync(current_, kPageSize, MS_ASYNC | MS_INVALIDATE);
+}
+
+void LogFileAccessor::MoveToEnd() {
+ Unmap(current_);
+ struct stat info;
+ if (fstat(fd_, &info) == -1) {
+ LOG(FATAL, "fstat(%d, %p) failed with %d: %s\n", fd_, &info, errno,
+ strerror(errno));
+ }
+ offset_ = info.st_size - kPageSize;
+ MapNextPage();
+}
+
+void LogFileAccessor::MapNextPage() {
+ if (writable_) {
+ if (ftruncate(fd_, offset_ + kPageSize) == -1) {
+ fprintf(stderr, "ftruncate(%d, %zd) failed with %d: %s. aborting\n",
+ fd_, kPageSize, errno, strerror(errno));
+ printf("see stderr\n");
+ abort();
+ }
+ }
+ current_ = static_cast<char *>(
+ mmap(NULL, kPageSize, PROT_READ | (writable_ ? PROT_WRITE : 0),
+ MAP_SHARED, fd_, offset_));
+ if (current_ == MAP_FAILED) {
+ LOG(FATAL,
+ "mmap(NULL, %zd, PROT_READ | PROT_WRITE, MAP_SHARED, %d, %jd)"
+ " failed with %d: %s. aborting\n",
+ kPageSize, fd_, static_cast<intmax_t>(offset_), errno,
+ strerror(errno));
+ }
+ offset_ += kPageSize;
+}
+
+void LogFileAccessor::Unmap(void *location) {
+ if (munmap(location, kPageSize) == -1) {
+ LOG(FATAL, "munmap(%p, %zd) failed with %d: %s. aborting\n", location,
+ kPageSize, errno, strerror(errno));
+ }
+}
+
+} // namespace linux_code
+} // namespace logging
+} // namespace aos
diff --git a/aos/linux_code/logging/binary_log_file.h b/aos/linux_code/logging/binary_log_file.h
index 6ce20cd..83f8dde 100644
--- a/aos/linux_code/logging/binary_log_file.h
+++ b/aos/linux_code/logging/binary_log_file.h
@@ -1,13 +1,9 @@
#ifndef AOS_LINUX_CODE_LOGGING_BINARY_LOG_FILE_H_
#define AOS_LINUX_CODE_LOGGING_BINARY_LOG_FILE_H_
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <sys/mman.h>
#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
+#include <stddef.h>
+#include <stdint.h>
#include <algorithm>
@@ -15,30 +11,47 @@
namespace aos {
namespace logging {
+namespace linux_code {
+
+// What to align messages to. A macro because it gets used in attributes.
+// This definition gets #undefed later. Use LogFileAccessor::kAlignment instead.
+#define MESSAGE_ALIGNMENT 8
// File format: {
// LogFileMessageHeader header;
// char *name; // of the process that wrote the message
-// char *message;
-// } not crossing kPageSize boundaries into the file.
+// void *message;
+// } not crossing kPageSize boundaries into the file and aligned to
+// MESSAGE_ALIGNMENT.
//
// Field sizes designed to fit the various values from LogMessage even on
-// other machines (hopefully) because they're baked into the files.
+// other machines (hopefully) because they're baked into the files. They are
+// layed out so that all of the fields are aligned even though the whole thing
+// is packed.
//
// A lot of the fields don't have comments because they're the same as the
// identically named fields in LogMessage.
-struct __attribute__((aligned)) LogFileMessageHeader {
- // gets futex_set once this one has been written
- // for readers keeping up with a live writer
+struct __attribute__((aligned(MESSAGE_ALIGNMENT))) __attribute__((packed))
+ LogFileMessageHeader {
+ // Represents the type of an individual message.
+ enum class MessageType : uint16_t {
+ // '\0'-terminated string.
+ kString,
+ kStructType,
+ kStruct,
+ };
+
+ // Gets futex_set once this one has been written
+ // for readers keeping up with a live writer.
//
- // gets initialized to 0 by ftruncate
+ // Gets initialized to 0 by ftruncate.
//
- // there will be something here after the last log on a "page" set to 2
- // (by the futex_set) to indicate that the next log is on the next page
+ // There will be something here after the last message on a "page" set to 2
+ // (by the futex_set) to indicate that the next message is on the next page.
mutex marker;
static_assert(sizeof(marker) == 4, "mutex changed size!");
- log_level level;
- static_assert(sizeof(level) == 1, "log_level changed size!");
+ static_assert(MESSAGE_ALIGNMENT >= alignof(mutex),
+ "MESSAGE_ALIGNMENT is too small");
uint32_t time_sec;
static_assert(sizeof(time_sec) >= sizeof(LogMessage::seconds),
@@ -49,150 +62,69 @@
int32_t source;
static_assert(sizeof(source) >= sizeof(LogMessage::source), "PIDs won't fit");
+
+ // Both including all of the bytes in that part of the message (ie any '\0's
+ // etc).
+ uint32_t name_size, message_size;
+
uint16_t sequence;
static_assert(sizeof(sequence) == sizeof(LogMessage::sequence),
"something changed");
- // both including the terminating '\0'
- uint32_t name_size;
- uint32_t message_size;
+ MessageType type;
+
+ log_level level;
+ static_assert(sizeof(level) == 1, "log_level changed size!");
};
static_assert(std::is_pod<LogFileMessageHeader>::value,
"LogFileMessageHeader will to get dumped to a file");
+static_assert(offsetof(LogFileMessageHeader, marker) == 0,
+ "marker has to be at the start so readers can find it");
// Handles the mmapping and munmapping for reading and writing log files.
class LogFileAccessor {
+ public:
+ LogFileAccessor(int fd, bool writable);
+
+ // message_size should be the total number of bytes needed for the message.
+ LogFileMessageHeader *GetWritePosition(size_t message_size);
+ // May return NULL iff wait is false.
+ const LogFileMessageHeader *ReadNextMessage(bool wait);
+
+ // Asynchronously syncs all open mappings.
+ void Sync() const;
+
+ // TODO(brians): This won't work any more.
+ void MoveToEnd();
+
private:
// The size of the chunks that get mmaped/munmapped together. Large enough so
// that not too much space is wasted and it's hopefully bigger than and a
- // multiple of the system page size but small enough so that really large chunks
- // of memory don't have to get mapped at the same time.
- static const size_t kPageSize = 32768;
- // What to align messages to. Necessary for futexes to work.
- static const size_t kAlignment = 64;
- static_assert(kAlignment >= __alignof__(mutex), "futexes will complain");
+ // multiple of the system page size but small enough so that really large
+ // chunks of memory don't have to get mapped at the same time.
+ static const size_t kPageSize = 16384;
+ // What to align messages to, copied into an actual constant.
+ static const size_t kAlignment = MESSAGE_ALIGNMENT;
+#undef MESSAGE_ALIGNMENT
const int fd_;
const bool writable_;
- off_t offset_; // into the file. will be aligned to kPageSize
+ // Into the file. Always a multiple of kPageSize.
+ off_t offset_;
char *current_;
size_t position_;
- inline unsigned long SystemPageSize() {
- static unsigned long r = sysconf(_SC_PAGESIZE);
- return r;
- }
- void MapNextPage() {
- if (writable_) {
- if (ftruncate(fd_, offset_ + kPageSize) == -1) {
- fprintf(stderr, "ftruncate(%d, %zd) failed with %d: %s. aborting\n",
- fd_, kPageSize, errno, strerror(errno));
- printf("see stderr\n");
- abort();
- }
- }
- current_ = static_cast<char *>(mmap(NULL, kPageSize,
- PROT_READ | (writable_ ? PROT_WRITE : 0),
- MAP_SHARED, fd_, offset_));
- if (current_ == MAP_FAILED) {
- fprintf(stderr, "mmap(NULL, %zd, PROT_READ | PROT_WRITE, MAP_SHARED, %d, %jd)"
- " failed with %d: %s. aborting\n", kPageSize, fd_,
- static_cast<intmax_t>(offset_), errno, strerror(errno));
- printf("see stderr\n");
- abort();
- }
- offset_ += kPageSize;
- }
- void Unmap(void *location) {
- if (munmap(location, kPageSize) == -1) {
- fprintf(stderr, "munmap(%p, %zd) failed with %d: %s. aborting\n",
- location, kPageSize, errno, strerror(errno));
- printf("see stderr\n");
- abort();
- }
- }
- public:
- LogFileAccessor(int fd, bool writable) : fd_(fd), writable_(writable),
- offset_(0), current_(0), position_(0) {
- // check to make sure that mmap will allow mmaping in chunks of kPageSize
- if (SystemPageSize() > kPageSize || (kPageSize % SystemPageSize()) != 0) {
- fprintf(stderr, "LogFileCommon: system page size (%lu)"
- " not compatible with kPageSize (%zd). aborting\n",
- SystemPageSize(), kPageSize);
- printf("see stderr\n");
- abort();
- }
+ void MapNextPage();
+ void Unmap(void *location);
- MapNextPage();
- }
- // message_size should be the total number of bytes needed for the message
- LogFileMessageHeader *GetWritePosition(size_t message_size) {
- if (position_ + message_size + (kAlignment - (message_size % kAlignment)) +
- sizeof(mutex) > kPageSize) {
- char *const temp = current_;
- MapNextPage();
- if (futex_set_value(static_cast<mutex *>(static_cast<void *>(
- &temp[position_])), 2) == -1) {
- fprintf(stderr,
- "LogFileCommon: futex_set_value(%p, 2) failed with %d: %s."
- " readers will hang\n",
- &temp[position_], errno, strerror(errno));
- }
- Unmap(temp);
- position_ = 0;
- }
- LogFileMessageHeader *const r = static_cast<LogFileMessageHeader *>(
- static_cast<void *>(¤t_[position_]));
- position_ += message_size;
- // keep it aligned for next time
+ // Advances position to the next (aligned) location.
+ void AlignPosition() {
position_ += kAlignment - (position_ % kAlignment);
- return r;
- }
- // may only return NULL if wait is false
- const LogFileMessageHeader *ReadNextMessage(bool wait) {
- LogFileMessageHeader *r;
- do {
- r = static_cast<LogFileMessageHeader *>(
- static_cast<void *>(¤t_[position_]));
- if (wait) {
- if (futex_wait(&r->marker) != 0) continue;
- }
- if (r->marker == 2) {
- Unmap(current_);
- MapNextPage();
- position_ = 0;
- r = static_cast<LogFileMessageHeader *>(static_cast<void *>(current_));
- }
- } while (wait && r->marker == 0);
- if (r->marker == 0) {
- return NULL;
- }
- position_ += sizeof(LogFileMessageHeader) + r->name_size + r->message_size;
- // keep it aligned for next time
- position_ += kAlignment - (position_ % kAlignment);
- return r;
- }
-
- // asynchronously syncs all open mappings
- void Sync() {
- msync(current_, kPageSize, MS_ASYNC | MS_INVALIDATE);
- }
-
- void MoveToEnd() {
- Unmap(current_);
- struct stat info;
- if (fstat(fd_, &info) == -1) {
- fprintf(stderr, "LOgFileCommon: fstat(%d, %p) failed with %d: %s\n",
- fd_, &info, errno, strerror(errno));
- printf("see stderr\n");
- abort();
- }
- offset_ = info.st_size - kPageSize;
- MapNextPage();
}
};
+} // namespace linux_code
} // namespace logging
} // namespace aos
diff --git a/aos/linux_code/logging/log_displayer.cc b/aos/linux_code/logging/log_displayer.cc
index 988c307..dc27c80 100644
--- a/aos/linux_code/logging/log_displayer.cc
+++ b/aos/linux_code/logging/log_displayer.cc
@@ -137,15 +137,18 @@
filename, strerror(errno));
exit(EXIT_FAILURE);
}
- ::aos::logging::LogFileAccessor accessor(fd, false);
+ ::aos::logging::linux_code::LogFileAccessor accessor(fd, false);
if (!start_at_beginning) {
accessor.MoveToEnd();
}
- const ::aos::logging::LogFileMessageHeader *msg;
+ const ::aos::logging::linux_code::LogFileMessageHeader *msg;
::aos::logging::LogMessage log_message;
do {
msg = accessor.ReadNextMessage(follow);
- if (msg == NULL) continue;
+ if (msg == NULL) {
+ fputs("reached end of file\n", stderr);
+ return 0;
+ }
if (::aos::logging::log_gt_important(filter_level, msg->level)) continue;
if (filter_name != NULL &&
strcmp(filter_name,
diff --git a/aos/linux_code/logging/log_streamer.cc b/aos/linux_code/logging/log_streamer.cc
index ad351e7..b6813e9 100644
--- a/aos/linux_code/logging/log_streamer.cc
+++ b/aos/linux_code/logging/log_streamer.cc
@@ -11,7 +11,6 @@
#include <inttypes.h>
#include "aos/linux_code/logging/linux_logging.h"
-#include "aos/linux_code/logging/binary_log_file.h"
#include "aos/linux_code/init.h"
#include "aos/linux_code/ipc_lib/queue.h"
#include "aos/common/logging/logging_impl.h"
diff --git a/aos/linux_code/logging/logging.gyp b/aos/linux_code/logging/logging.gyp
index 96c197f..95459f3 100644
--- a/aos/linux_code/logging/logging.gyp
+++ b/aos/linux_code/logging/logging.gyp
@@ -1,5 +1,6 @@
{
'targets': [
+ # linux_* is dealt with by aos/build/aos.gyp:logging.
{
'target_name': 'binary_log_writer',
'type': 'executable',
@@ -10,6 +11,7 @@
'<(AOS)/build/aos.gyp:logging',
'<(AOS)/linux_code/linux_code.gyp:init',
'<(AOS)/linux_code/linux_code.gyp:configuration',
+ 'binary_log_file',
],
},
{
@@ -34,6 +36,20 @@
'dependencies': [
'<(AOS)/build/aos.gyp:logging',
'<(AOS)/linux_code/linux_code.gyp:init',
+ 'binary_log_file',
+ ],
+ },
+ {
+ 'target_name': 'binary_log_file',
+ 'type': 'static_library',
+ 'sources': [
+ 'binary_log_file.cc',
+ ],
+ 'dependencies': [
+ '<(AOS)/build/aos.gyp:logging',
+ ],
+ 'export_dependent_settings': [
+ '<(AOS)/build/aos.gyp:logging',
],
},
],