Make logfile decoders accept a nested decoder instead of filename.
Change-Id: I4685d1dfbbd2bdd336762ee67471fa275883b772
Signed-off-by: Tyler Chatow <tyler.chatow@bluerivertech.com>
diff --git a/aos/events/logging/buffer_encoder.cc b/aos/events/logging/buffer_encoder.cc
index 10a7ed1..6ef61d4 100644
--- a/aos/events/logging/buffer_encoder.cc
+++ b/aos/events/logging/buffer_encoder.cc
@@ -39,7 +39,7 @@
}
DummyDecoder::DummyDecoder(std::string_view filename)
- : fd_(open(std::string(filename).c_str(), O_RDONLY | O_CLOEXEC)) {
+ : filename_(filename), fd_(open(filename_.c_str(), O_RDONLY | O_CLOEXEC)) {
PCHECK(fd_ != -1) << ": Failed to open " << filename;
}
diff --git a/aos/events/logging/buffer_encoder.h b/aos/events/logging/buffer_encoder.h
index 1eddd00..235a49c 100644
--- a/aos/events/logging/buffer_encoder.h
+++ b/aos/events/logging/buffer_encoder.h
@@ -2,9 +2,8 @@
#define AOS_EVENTS_LOGGING_BUFFER_ENCODER_H_
#include "absl/types/span.h"
-#include "flatbuffers/flatbuffers.h"
-
#include "aos/events/logging/logger_generated.h"
+#include "flatbuffers/flatbuffers.h"
namespace aos::logger {
@@ -89,6 +88,9 @@
// Returns less than end-begin if all bytes have been read. Otherwise, this
// will always fill the whole range.
virtual size_t Read(uint8_t *begin, uint8_t *end) = 0;
+
+ // Returns the underlying filename, for debugging purposes.
+ virtual std::string_view filename() const = 0;
};
// Simply reads the contents of the file into the target buffer.
@@ -102,8 +104,11 @@
~DummyDecoder() override;
size_t Read(uint8_t *begin, uint8_t *end) final;
+ std::string_view filename() const final { return filename_; }
private:
+ const std::string filename_;
+
// File descriptor for the log file.
int fd_;
diff --git a/aos/events/logging/logfile_utils.cc b/aos/events/logging/logfile_utils.cc
index cd2ceb8..4c19867 100644
--- a/aos/events/logging/logfile_utils.cc
+++ b/aos/events/logging/logfile_utils.cc
@@ -17,9 +17,9 @@
#include "glog/logging.h"
#if defined(__x86_64__)
-#define ENABLE_LZMA 1
+#define ENABLE_LZMA (!__has_feature(memory_sanitizer))
#elif defined(__aarch64__)
-#define ENABLE_LZMA 1
+#define ENABLE_LZMA (!__has_feature(memory_sanitizer))
#else
#define ENABLE_LZMA 0
#endif
@@ -317,15 +317,15 @@
}
SpanReader::SpanReader(std::string_view filename) : filename_(filename) {
- static const std::string_view kXz = ".xz";
+ decoder_ = std::make_unique<DummyDecoder>(filename);
+
+ static constexpr std::string_view kXz = ".xz";
if (filename.substr(filename.size() - kXz.size()) == kXz) {
#if ENABLE_LZMA
- decoder_ = std::make_unique<ThreadedLzmaDecoder>(filename);
+ decoder_ = std::make_unique<ThreadedLzmaDecoder>(std::move(decoder_));
#else
LOG(FATAL) << "Reading xz-compressed files not supported on this platform";
#endif
- } else {
- decoder_ = std::make_unique<DummyDecoder>(filename);
}
}
diff --git a/aos/events/logging/lzma_encoder.cc b/aos/events/logging/lzma_encoder.cc
index 60a203e..1024b91 100644
--- a/aos/events/logging/lzma_encoder.cc
+++ b/aos/events/logging/lzma_encoder.cc
@@ -149,8 +149,9 @@
total_bytes_ += last_avail_out - stream_.avail_out;
}
-LzmaDecoder::LzmaDecoder(std::string_view filename)
- : dummy_decoder_(filename), stream_(LZMA_STREAM_INIT), filename_(filename) {
+LzmaDecoder::LzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder)
+ : underlying_decoder_(std::move(underlying_decoder)),
+ stream_(LZMA_STREAM_INIT) {
compressed_data_.resize(kBufSize);
lzma_ret status =
@@ -174,8 +175,8 @@
while (stream_.avail_out > 0) {
if (action_ == LZMA_RUN && stream_.avail_in == 0) {
// Read more bytes from the file if we're all out.
- const size_t count =
- dummy_decoder_.Read(compressed_data_.begin(), compressed_data_.end());
+ const size_t count = underlying_decoder_->Read(compressed_data_.begin(),
+ compressed_data_.end());
if (count == 0) {
// No more data to read in the file, begin the finishing operation.
action_ = LZMA_FINISH;
@@ -196,17 +197,18 @@
// If we fail to decompress, give up. Return everything that has been
// produced so far.
- if (!LzmaCodeIsOk(status, filename_)) {
+ if (!LzmaCodeIsOk(status, filename())) {
finished_ = true;
- LOG(WARNING) << filename_ << " is truncated or corrupted.";
+ LOG(WARNING) << filename() << " is truncated or corrupted.";
return (end - begin) - stream_.avail_out;
}
}
return end - begin;
}
-ThreadedLzmaDecoder::ThreadedLzmaDecoder(std::string_view filename)
- : decoder_(filename), decode_thread_([this] {
+ThreadedLzmaDecoder::ThreadedLzmaDecoder(
+ std::unique_ptr<DataDecoder> underlying_decoder)
+ : decoder_(std::move(underlying_decoder)), decode_thread_([this] {
std::unique_lock lock(decode_mutex_);
while (true) {
// Wake if the queue is too small or we are finished.
diff --git a/aos/events/logging/lzma_encoder.h b/aos/events/logging/lzma_encoder.h
index 972ed6c..6696200 100644
--- a/aos/events/logging/lzma_encoder.h
+++ b/aos/events/logging/lzma_encoder.h
@@ -1,6 +1,7 @@
#ifndef AOS_EVENTS_LOGGING_LZMA_ENCODER_H_
#define AOS_EVENTS_LOGGING_LZMA_ENCODER_H_
+#include <string_view>
#include <condition_variable>
#include <mutex>
#include <thread>
@@ -51,7 +52,9 @@
// Decompresses data with liblzma.
class LzmaDecoder final : public DataDecoder {
public:
- explicit LzmaDecoder(std::string_view filename);
+ explicit LzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder);
+ explicit LzmaDecoder(std::string_view filename)
+ : LzmaDecoder(std::make_unique<DummyDecoder>(filename)) {}
LzmaDecoder(const LzmaDecoder &) = delete;
LzmaDecoder(LzmaDecoder &&other) = delete;
LzmaDecoder &operator=(const LzmaDecoder &) = delete;
@@ -59,6 +62,9 @@
~LzmaDecoder();
size_t Read(uint8_t *begin, uint8_t *end) final;
+ std::string_view filename() const final {
+ return underlying_decoder_->filename();
+ }
private:
// Size of temporary buffer to use.
@@ -67,7 +73,7 @@
// Temporary buffer for storing compressed data.
ResizeableBuffer compressed_data_;
// Used for reading data from the file.
- DummyDecoder dummy_decoder_;
+ std::unique_ptr<DataDecoder> underlying_decoder_;
// Stream for decompression.
lzma_stream stream_;
// The current action. This is LZMA_RUN until we've run out of data to read
@@ -76,9 +82,6 @@
// Flag that represents whether or not all the data from the file has been
// successfully decoded.
bool finished_ = false;
-
- // Filename we are decompressing.
- std::string filename_;
};
// Decompresses data with liblzma in a new thread, up to a maximum queue
@@ -86,7 +89,9 @@
// or block until more data is queued or the stream finishes.
class ThreadedLzmaDecoder : public DataDecoder {
public:
- explicit ThreadedLzmaDecoder(std::string_view filename);
+ explicit ThreadedLzmaDecoder(std::string_view filename)
+ : ThreadedLzmaDecoder(std::make_unique<DummyDecoder>(filename)) {}
+ explicit ThreadedLzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder);
ThreadedLzmaDecoder(const ThreadedLzmaDecoder &) = delete;
ThreadedLzmaDecoder &operator=(const ThreadedLzmaDecoder &) = delete;
@@ -94,6 +99,8 @@
size_t Read(uint8_t *begin, uint8_t *end) final;
+ std::string_view filename() const final { return decoder_.filename(); }
+
private:
static constexpr size_t kBufSize{256 * 1024};
static constexpr size_t kQueueSize{8};