Add a flag to quiet down log file sorting
When a bunch of part files get truncated, the resulting log file spam is
enough that the actual error gets dwarfed. The value isn't there. This
lets the user turn that off if they want.
Change-Id: I2a317dd1d74f8ef1ad55402d4cb732ee88c1b087
Signed-off-by: Austin Schuh <austin.linux@gmail.com>
diff --git a/aos/events/logging/logfile_sorting.cc b/aos/events/logging/logfile_sorting.cc
index 7eb17a0..ae13e5b 100644
--- a/aos/events/logging/logfile_sorting.cc
+++ b/aos/events/logging/logfile_sorting.cc
@@ -15,6 +15,9 @@
#include "openssl/sha.h"
#include "sys/stat.h"
+DEFINE_bool(quiet_sorting, false,
+ "If true, sort with minimal messages about truncated files.");
+
namespace aos {
namespace logger {
namespace chrono = std::chrono;
@@ -342,11 +345,13 @@
// resources, so close a big batch at once periodically.
part_readers.clear();
}
- part_readers.emplace_back(part);
+ part_readers.emplace_back(part, FLAGS_quiet_sorting);
std::optional<SizePrefixedFlatbufferVector<LogFileHeader>> log_header =
ReadHeader(&part_readers.back());
if (!log_header) {
- LOG(WARNING) << "Skipping " << part << " without a header";
+ if (!FLAGS_quiet_sorting) {
+ LOG(WARNING) << "Skipping " << part << " without a header";
+ }
corrupted.emplace_back(part);
continue;
}
@@ -461,7 +466,9 @@
std::optional<SizePrefixedFlatbufferVector<MessageHeader>> first_message =
ReadNthMessage(part, 0);
if (!first_message) {
- LOG(WARNING) << "Skipping " << part << " without any messages";
+ if (!FLAGS_quiet_sorting) {
+ LOG(WARNING) << "Skipping " << part << " without any messages";
+ }
corrupted.emplace_back(part);
continue;
}
diff --git a/aos/events/logging/logfile_utils.cc b/aos/events/logging/logfile_utils.cc
index 0293aa1..59b106e 100644
--- a/aos/events/logging/logfile_utils.cc
+++ b/aos/events/logging/logfile_utils.cc
@@ -336,15 +336,18 @@
return message_header_builder.Finish();
}
-SpanReader::SpanReader(std::string_view filename) : filename_(filename) {
+SpanReader::SpanReader(std::string_view filename, bool quiet)
+ : filename_(filename) {
decoder_ = std::make_unique<DummyDecoder>(filename);
static constexpr std::string_view kXz = ".xz";
static constexpr std::string_view kSnappy = SnappyDecoder::kExtension;
if (filename.substr(filename.size() - kXz.size()) == kXz) {
#if ENABLE_LZMA
- decoder_ = std::make_unique<ThreadedLzmaDecoder>(std::move(decoder_));
+ decoder_ =
+ std::make_unique<ThreadedLzmaDecoder>(std::move(decoder_), quiet);
#else
+ (void)quiet;
LOG(FATAL) << "Reading xz-compressed files not supported on this platform";
#endif
} else if (filename.substr(filename.size() - kSnappy.size()) == kSnappy) {
diff --git a/aos/events/logging/logfile_utils.h b/aos/events/logging/logfile_utils.h
index 3a84726..f2bdc42 100644
--- a/aos/events/logging/logfile_utils.h
+++ b/aos/events/logging/logfile_utils.h
@@ -202,7 +202,7 @@
// Class to read chunks out of a log file.
class SpanReader {
public:
- SpanReader(std::string_view filename);
+ SpanReader(std::string_view filename, bool quiet = false);
std::string_view filename() const { return filename_; }
diff --git a/aos/events/logging/lzma_encoder.cc b/aos/events/logging/lzma_encoder.cc
index e9915af..f32d7e2 100644
--- a/aos/events/logging/lzma_encoder.cc
+++ b/aos/events/logging/lzma_encoder.cc
@@ -149,9 +149,11 @@
total_bytes_ += last_avail_out - stream_.avail_out;
}
-LzmaDecoder::LzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder)
+LzmaDecoder::LzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder,
+ bool quiet)
: underlying_decoder_(std::move(underlying_decoder)),
- stream_(LZMA_STREAM_INIT) {
+ stream_(LZMA_STREAM_INIT),
+ quiet_(quiet) {
compressed_data_.resize(kBufSize);
lzma_ret status =
@@ -200,9 +202,13 @@
if (!LzmaCodeIsOk(status, filename())) {
finished_ = true;
if (status == LZMA_DATA_ERROR) {
- LOG(WARNING) << filename() << " is corrupted.";
+ if (!quiet_ || VLOG_IS_ON(1)) {
+ LOG(WARNING) << filename() << " is corrupted.";
+ }
} else if (status == LZMA_BUF_ERROR) {
- LOG(WARNING) << filename() << " is truncated or corrupted.";
+ if (!quiet_ || VLOG_IS_ON(1)) {
+ LOG(WARNING) << filename() << " is truncated or corrupted.";
+ }
} else {
LOG(FATAL) << "Unknown error " << status << " when reading "
<< filename();
@@ -214,8 +220,8 @@
}
ThreadedLzmaDecoder::ThreadedLzmaDecoder(
- std::unique_ptr<DataDecoder> underlying_decoder)
- : decoder_(std::move(underlying_decoder)), decode_thread_([this] {
+ std::unique_ptr<DataDecoder> underlying_decoder, bool quiet)
+ : decoder_(std::move(underlying_decoder), quiet), decode_thread_([this] {
std::unique_lock lock(decode_mutex_);
while (true) {
// Wake if the queue is too small or we are finished.
diff --git a/aos/events/logging/lzma_encoder.h b/aos/events/logging/lzma_encoder.h
index fc6fcb6..7b7010a 100644
--- a/aos/events/logging/lzma_encoder.h
+++ b/aos/events/logging/lzma_encoder.h
@@ -54,9 +54,10 @@
public:
static constexpr std::string_view kExtension = ".xz";
- explicit LzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder);
- explicit LzmaDecoder(std::string_view filename)
- : LzmaDecoder(std::make_unique<DummyDecoder>(filename)) {}
+ explicit LzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder,
+ bool quiet = false);
+ explicit LzmaDecoder(std::string_view filename, bool quiet = false)
+ : LzmaDecoder(std::make_unique<DummyDecoder>(filename), quiet) {}
LzmaDecoder(const LzmaDecoder &) = delete;
LzmaDecoder(LzmaDecoder &&other) = delete;
LzmaDecoder &operator=(const LzmaDecoder &) = delete;
@@ -84,6 +85,9 @@
// Flag that represents whether or not all the data from the file has been
// successfully decoded.
bool finished_ = false;
+ // Flag to signal how quiet to be when logging potential issues around
+ // truncation.
+ const bool quiet_ = false;
};
// Decompresses data with liblzma in a new thread, up to a maximum queue
@@ -91,9 +95,10 @@
// or block until more data is queued or the stream finishes.
class ThreadedLzmaDecoder : public DataDecoder {
public:
- explicit ThreadedLzmaDecoder(std::string_view filename)
- : ThreadedLzmaDecoder(std::make_unique<DummyDecoder>(filename)) {}
- explicit ThreadedLzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder);
+ explicit ThreadedLzmaDecoder(std::string_view filename, bool quiet = false)
+ : ThreadedLzmaDecoder(std::make_unique<DummyDecoder>(filename), quiet) {}
+ explicit ThreadedLzmaDecoder(std::unique_ptr<DataDecoder> underlying_decoder,
+ bool quiet = false);
ThreadedLzmaDecoder(const ThreadedLzmaDecoder &) = delete;
ThreadedLzmaDecoder &operator=(const ThreadedLzmaDecoder &) = delete;