Encode flatbuffers directly into the encoder when logging
We were running out of memory when running for many hours. Initial
debugging looked like it was a heap fragmentation issue. Tracking the
allocated memory using the malloc hooks wasn't showing any growth of
memory. The heap was growing though.
Instead of allocating a FlatBufferBuilder/DetachedBuffer for each
message to be logged, we can instead have the BufferEncoder provide
memory to write to, and have it only alloate that buffer space once, and
allocate it to the maximum size that a writer might see.
Change-Id: I046bd2422aea368867b0c63cee7d04c6033fe724
Signed-off-by: Austin Schuh <austin.linux@gmail.com>
diff --git a/aos/events/logging/logger_main.cc b/aos/events/logging/logger_main.cc
index 526eb31..81b4e3f 100644
--- a/aos/events/logging/logger_main.cc
+++ b/aos/events/logging/logger_main.cc
@@ -3,6 +3,9 @@
#include "aos/configuration.h"
#include "aos/events/logging/log_writer.h"
+#ifdef LZMA
+#include "aos/events/logging/lzma_encoder.h"
+#endif
#include "aos/events/logging/snappy_encoder.h"
#include "aos/events/shm_event_loop.h"
#include "aos/init.h"
@@ -18,9 +21,17 @@
DEFINE_bool(snappy_compress, false, "If true, compress log data using snappy.");
+#ifdef LZMA
+DEFINE_bool(xz_compress, false, "If true, compress log data using xz.");
+#endif
+
DEFINE_double(rotate_every, 0.0,
"If set, rotate the logger after this many seconds");
+#ifdef LZMA
+DEFINE_int32(xz_compression_level, 9, "Compression level for the LZMA Encoder");
+#endif
+
int main(int argc, char *argv[]) {
gflags::SetUsageMessage(
"This program provides a simple logger binary that logs all SHMEM data "
@@ -41,8 +52,17 @@
if (FLAGS_snappy_compress) {
log_namer->set_extension(aos::logger::SnappyDecoder::kExtension);
- log_namer->set_encoder_factory(
- []() { return std::make_unique<aos::logger::SnappyEncoder>(); });
+ log_namer->set_encoder_factory([](size_t max_message_size) {
+ return std::make_unique<aos::logger::SnappyEncoder>(max_message_size);
+ });
+#ifdef LZMA
+ } else if (FLAGS_xz_compress) {
+ log_namer->set_extension(aos::logger::LzmaEncoder::kExtension);
+ log_namer->set_encoder_factory([](size_t max_message_size) {
+ return std::make_unique<aos::logger::LzmaEncoder>(
+ max_message_size, FLAGS_xz_compression_level);
+ });
+#endif
}
aos::monotonic_clock::time_point last_rotation_time =