got binary logging actually fully working (!!!)
diff --git a/aos/linux_code/logging/binary_log_file.h b/aos/linux_code/logging/binary_log_file.h
index 83f8dde..58aa792 100644
--- a/aos/linux_code/logging/binary_log_file.h
+++ b/aos/linux_code/logging/binary_log_file.h
@@ -35,7 +35,7 @@
LogFileMessageHeader {
// Represents the type of an individual message.
enum class MessageType : uint16_t {
- // '\0'-terminated string.
+ // char[] (no '\0' on the end).
kString,
kStructType,
kStruct,
@@ -63,8 +63,7 @@
int32_t source;
static_assert(sizeof(source) >= sizeof(LogMessage::source), "PIDs won't fit");
- // Both including all of the bytes in that part of the message (ie any '\0's
- // etc).
+ // Both including all of the bytes in that part of the message.
uint32_t name_size, message_size;
uint16_t sequence;
diff --git a/aos/linux_code/logging/binary_log_writer.cc b/aos/linux_code/logging/binary_log_writer.cc
index 22c635e..aa81521 100644
--- a/aos/linux_code/logging/binary_log_writer.cc
+++ b/aos/linux_code/logging/binary_log_writer.cc
@@ -10,17 +10,52 @@
#include <fcntl.h>
#include <map>
+#include <unordered_set>
#include "aos/linux_code/logging/linux_logging.h"
#include "aos/linux_code/logging/binary_log_file.h"
#include "aos/linux_code/init.h"
#include "aos/linux_code/configuration.h"
+#include "aos/common/queue_types.h"
namespace aos {
namespace logging {
namespace linux_code {
namespace {
+void CheckTypeWritten(uint32_t type_id, LogFileAccessor &writer) {
+ static ::std::unordered_set<uint32_t> written_type_ids;
+ if (written_type_ids.count(type_id) > 0) return;
+ if (MessageType::IsPrimitive(type_id)) return;
+
+ const MessageType &type = type_cache::Get(type_id);
+ for (int i = 0; i < type.number_fields; ++i) {
+ CheckTypeWritten(type.fields[i]->type, writer);
+ }
+
+ char buffer[1024];
+ ssize_t size = type.Serialize(buffer, sizeof(buffer));
+ if (size == -1) {
+ LOG(WARNING, "%zu-byte buffer is too small to serialize type %s\n",
+ sizeof(buffer), type.name.c_str());
+ return;
+ }
+ LogFileMessageHeader *const output =
+ writer.GetWritePosition(sizeof(LogFileMessageHeader) + size);
+
+ output->time_sec = output->time_nsec = 0;
+ output->source = getpid();
+ output->name_size = 0;
+ output->sequence = 0;
+ output->level = FATAL;
+
+ memcpy(output + 1, buffer, size);
+ output->message_size = size;
+
+ output->type = LogFileMessageHeader::MessageType::kStructType;
+ futex_set(&output->marker);
+}
+
int BinaryLogReaderMain() {
InitNRT();
@@ -81,23 +116,52 @@
const LogMessage *const msg = ReadNext();
if (msg == NULL) continue;
- // add 1 for terminating '\0'
- size_t name_size = strlen(msg->name) + 1;
- size_t message_size = strlen(msg->message) + 1;
-
- LogFileMessageHeader *const output = writer.GetWritePosition(
- sizeof(LogFileMessageHeader) + name_size + message_size);
+ size_t output_length =
+ sizeof(LogFileMessageHeader) + msg->name_length + msg->message_length;
+ if (msg->type == LogMessage::Type::kStruct) {
+ output_length += sizeof(msg->structure.type_id) + sizeof(uint32_t) +
+ msg->structure.string_length;
+ CheckTypeWritten(msg->structure.type_id, writer);
+ }
+ LogFileMessageHeader *const output = writer.GetWritePosition(output_length);;
char *output_strings = reinterpret_cast<char *>(output) + sizeof(*output);
- output->name_size = name_size;
- output->message_size = message_size;
+ output->name_size = msg->name_length;
+ output->message_size = msg->message_length;
output->source = msg->source;
output->level = msg->level;
output->time_sec = msg->seconds;
output->time_nsec = msg->nseconds;
output->sequence = msg->sequence;
- output->type = LogFileMessageHeader::MessageType::kString;
- memcpy(output_strings, msg->name, name_size);
- memcpy(output_strings + name_size, msg->message, message_size);
+ memcpy(output_strings, msg->name, msg->name_length);
+
+ switch (msg->type) {
+ case LogMessage::Type::kString:
+ memcpy(output_strings + msg->name_length, msg->message,
+ msg->message_length);
+ output->type = LogFileMessageHeader::MessageType::kString;
+ break;
+ case LogMessage::Type::kStruct:
+ char *position = output_strings + msg->name_length;
+
+ memcpy(position, &msg->structure.type_id, sizeof(msg->structure.type_id));
+ position += sizeof(msg->structure.type_id);
+ output->message_size += sizeof(msg->structure.type_id);
+
+ uint32_t length = msg->structure.string_length;
+ memcpy(position, &length, sizeof(length));
+ position += sizeof(length);
+ memcpy(position, msg->structure.serialized, length);
+ position += length;
+ output->message_size += sizeof(length) + length;
+
+ memcpy(position,
+ msg->structure.serialized + msg->structure.string_length,
+ msg->message_length);
+
+ output->type = LogFileMessageHeader::MessageType::kStruct;
+ break;
+ }
+
futex_set(&output->marker);
logging::linux_code::Free(msg);
diff --git a/aos/linux_code/logging/linux_logging.cc b/aos/linux_code/logging/linux_logging.cc
index 985577b..676e3b7 100644
--- a/aos/linux_code/logging/linux_logging.cc
+++ b/aos/linux_code/logging/linux_logging.cc
@@ -25,15 +25,29 @@
namespace linux_code {
namespace {
-class linuxQueueLogImplementation : public LogImplementation {
- virtual void DoLog(log_level level, const char *format, va_list ap) {
+class LinuxQueueLogImplementation : public LogImplementation {
+ LogMessage *GetMessageOrDie() {
LogMessage *message = static_cast<LogMessage *>(queue->GetMessage());
if (message == NULL) {
- LOG(FATAL, "queue get message failed\n");
+ LOG(FATAL, "%p->GetMessage() failed\n", queue);
+ } else {
+ return message;
}
+ }
+ virtual void DoLog(log_level level, const char *format, va_list ap) override {
+ LogMessage *message = GetMessageOrDie();
internal::FillInMessage(level, format, ap, message);
+ Write(message);
+ }
+ virtual void LogStruct(log_level level, const ::std::string &message_string,
+ size_t size, const MessageType *type,
+ const ::std::function<size_t(char *)> &serialize)
+ override {
+ LogMessage *message = GetMessageOrDie();
+ internal::FillInMessageStructure(level, message_string, size, type,
+ serialize, message);
Write(message);
}
};
@@ -48,7 +62,7 @@
Die("logging: couldn't fetch queue\n");
}
- AddImplementation(new linuxQueueLogImplementation());
+ AddImplementation(new LinuxQueueLogImplementation());
}
const LogMessage *ReadNext(int flags, int *index) {
diff --git a/aos/linux_code/logging/log_displayer.cc b/aos/linux_code/logging/log_displayer.cc
index dc27c80..7b56f35 100644
--- a/aos/linux_code/logging/log_displayer.cc
+++ b/aos/linux_code/logging/log_displayer.cc
@@ -7,8 +7,13 @@
#include <inttypes.h>
#include <errno.h>
+#include <algorithm>
+
#include "aos/linux_code/logging/binary_log_file.h"
#include "aos/common/logging/logging_impl.h"
+#include "aos/common/queue_types.h"
+
+using ::aos::logging::linux_code::LogFileMessageHeader;
namespace {
@@ -45,6 +50,7 @@
int main(int argc, char **argv) {
const char *filter_name = NULL;
+ size_t filter_length;
log_level filter_level = INFO;
bool follow = false, start_at_beginning = true;
const char *filename = "aos_log-current";
@@ -80,6 +86,7 @@
abort();
case 'n':
filter_name = optarg;
+ filter_length = strlen(filter_name);
break;
case 'l':
filter_level = ::aos::logging::str_log(optarg);
@@ -141,7 +148,8 @@
if (!start_at_beginning) {
accessor.MoveToEnd();
}
- const ::aos::logging::linux_code::LogFileMessageHeader *msg;
+
+ const LogFileMessageHeader *msg;
::aos::logging::LogMessage log_message;
do {
msg = accessor.ReadNextMessage(follow);
@@ -149,25 +157,72 @@
fputs("reached end of file\n", stderr);
return 0;
}
- if (::aos::logging::log_gt_important(filter_level, msg->level)) continue;
- if (filter_name != NULL &&
- strcmp(filter_name,
- reinterpret_cast<const char *>(msg) + sizeof(*msg)) != 0) {
+
+ if (msg->type == LogFileMessageHeader::MessageType::kStructType) {
+ size_t bytes = msg->message_size;
+ ::aos::MessageType *type = ::aos::MessageType::Deserialize(
+ reinterpret_cast<const char *>(msg + 1), &bytes);
+ ::aos::type_cache::Add(*type);
continue;
}
+ if (::aos::logging::log_gt_important(filter_level, msg->level)) continue;
+ if (filter_name != NULL) {
+ if (filter_length != msg->name_size) continue;
+ if (memcmp(filter_name,
+ reinterpret_cast<const char *>(msg) + sizeof(*msg),
+ filter_length) !=
+ 0) {
+ continue;
+ }
+ }
+
log_message.source = msg->source;
log_message.sequence = msg->sequence;
log_message.level = msg->level;
log_message.seconds = msg->time_sec;
log_message.nseconds = msg->time_nsec;
- strncpy(log_message.name,
- reinterpret_cast<const char *>(msg) + sizeof(*msg),
- sizeof(log_message.name));
- strncpy(log_message.message,
- reinterpret_cast<const char *>(msg) + sizeof(*msg) +
- msg->name_size,
- sizeof(log_message.message));
+ memcpy(log_message.name, reinterpret_cast<const char *>(msg) + sizeof(*msg),
+ ::std::min<size_t>(sizeof(log_message.name), msg->name_size));
+ log_message.message_length = msg->message_size;
+ log_message.name_length = msg->name_size;
+
+ switch (msg->type) {
+ case LogFileMessageHeader::MessageType::kStruct: {
+ const char *position =
+ reinterpret_cast<const char *>(msg + 1) + msg->name_size;
+ memcpy(&log_message.structure.type_id, position,
+ sizeof(log_message.structure.type_id));
+ position += sizeof(log_message.structure.type_id);
+
+ uint32_t length;
+ memcpy(&length, position, sizeof(length));
+ log_message.structure.string_length = length;
+ position += sizeof(length);
+ memcpy(log_message.structure.serialized, position, length);
+ position += length;
+
+ log_message.message_length -=
+ sizeof(log_message.structure.type_id) + sizeof(uint32_t) +
+ log_message.structure.string_length;
+ memcpy(log_message.structure.serialized +
+ log_message.structure.string_length,
+ position, log_message.message_length);
+
+ log_message.type = ::aos::logging::LogMessage::Type::kStruct;
+ break;
+ }
+ case LogFileMessageHeader::MessageType::kString:
+ memcpy(
+ log_message.message,
+ reinterpret_cast<const char *>(msg) + sizeof(*msg) + msg->name_size,
+ ::std::min<size_t>(sizeof(log_message.message), msg->message_size));
+ log_message.type = ::aos::logging::LogMessage::Type::kString;
+ break;
+ case LogFileMessageHeader::MessageType::kStructType:
+ LOG(FATAL, "shouldn't get here\n");
+ break;
+ };
::aos::logging::internal::PrintMessage(stdout, log_message);
} while (msg != NULL);
}
diff --git a/aos/linux_code/logging/logging.gyp b/aos/linux_code/logging/logging.gyp
index 95459f3..32df43d 100644
--- a/aos/linux_code/logging/logging.gyp
+++ b/aos/linux_code/logging/logging.gyp
@@ -12,6 +12,7 @@
'<(AOS)/linux_code/linux_code.gyp:init',
'<(AOS)/linux_code/linux_code.gyp:configuration',
'binary_log_file',
+ '<(AOS)/common/common.gyp:queue_types',
],
},
{
@@ -37,6 +38,7 @@
'<(AOS)/build/aos.gyp:logging',
'<(AOS)/linux_code/linux_code.gyp:init',
'binary_log_file',
+ '<(AOS)/common/common.gyp:queue_types',
],
},
{