Make log file header comparison more efficient
Use FlatbufferVector instead of DetachedBuffer because the resulting
copy is significantly more efficient than the reflection based copy that
was being done before.
Change-Id: I9721e014bb6911ae9cde8586313f8b511ff06c8f
diff --git a/aos/events/logging/logfile_utils.cc b/aos/events/logging/logfile_utils.cc
index 123c579..c78b160 100644
--- a/aos/events/logging/logfile_utils.cc
+++ b/aos/events/logging/logfile_utils.cc
@@ -264,7 +264,8 @@
absl::Span<const uint8_t> config_data = span_reader.ReadMessage();
// Make sure something was read.
- CHECK(config_data != absl::Span<const uint8_t>());
+ CHECK(config_data != absl::Span<const uint8_t>())
+ << ": Failed to read header from: " << filename;
// And copy the config so we have it forever.
std::vector<uint8_t> data(
@@ -273,15 +274,20 @@
}
MessageReader::MessageReader(std::string_view filename)
- : span_reader_(filename) {
+ : span_reader_(filename),
+ raw_log_file_header_(FlatbufferVector<LogFileHeader>::Empty()) {
// Make sure we have enough to read the size.
- absl::Span<const uint8_t> config_data = span_reader_.ReadMessage();
+ absl::Span<const uint8_t> header_data = span_reader_.ReadMessage();
// Make sure something was read.
- CHECK(config_data != absl::Span<const uint8_t>());
+ CHECK(header_data != absl::Span<const uint8_t>())
+ << ": Failed to read header from: " << filename;
- // And copy the config so we have it forever.
- configuration_ = std::vector<uint8_t>(config_data.begin(), config_data.end());
+ // And copy the header data so we have it forever.
+ std::vector<uint8_t> header_data_copy(
+ header_data.begin() + sizeof(flatbuffers::uoffset_t), header_data.end());
+ raw_log_file_header_ =
+ FlatbufferVector<LogFileHeader>(std::move(header_data_copy));
max_out_of_order_duration_ =
std::chrono::nanoseconds(log_file_header()->max_out_of_order_duration());
@@ -310,12 +316,12 @@
SplitMessageReader::SplitMessageReader(
const std::vector<std::string> &filenames)
: filenames_(filenames),
- log_file_header_(FlatbufferDetachedBuffer<LogFileHeader>::Empty()) {
+ log_file_header_(FlatbufferVector<LogFileHeader>::Empty()) {
CHECK(NextLogFile()) << ": filenames is empty. Need files to read.";
// Grab any log file header. They should all match (and we will check as we
// open more of them).
- log_file_header_ = CopyFlatBuffer(message_reader_->log_file_header());
+ log_file_header_ = message_reader_->raw_log_file_header();
// Setup per channel state.
channels_.resize(configuration()->channels()->size());
@@ -362,8 +368,8 @@
// We can't support the config diverging between two log file headers. See if
// they are the same.
if (next_filename_index_ != 0) {
- CHECK(CompareFlatBuffer(&log_file_header_.message(),
- message_reader_->log_file_header()))
+ CHECK(CompareFlatBuffer(message_reader_->raw_log_file_header(),
+ log_file_header_))
<< ": Header is different between log file chunks "
<< filenames_[next_filename_index_] << " and "
<< filenames_[next_filename_index_ - 1] << ", this is not supported.";
@@ -980,8 +986,7 @@
ChannelMerger::ChannelMerger(
const std::vector<std::vector<std::string>> &filenames)
: split_message_readers_(MakeSplitMessageReaders(filenames)),
- log_file_header_(
- CopyFlatBuffer(split_message_readers_[0]->log_file_header())) {
+ log_file_header_(split_message_readers_[0]->raw_log_file_header()) {
// Now, confirm that the configuration matches for each and pick a start time.
// Also return the list of possible nodes.
for (const std::unique_ptr<SplitMessageReader> &reader :
diff --git a/aos/events/logging/logfile_utils.h b/aos/events/logging/logfile_utils.h
index 7acbbd3..b1a2d67 100644
--- a/aos/events/logging/logfile_utils.h
+++ b/aos/events/logging/logfile_utils.h
@@ -3,11 +3,15 @@
#include <sys/uio.h>
+#include <chrono>
#include <deque>
+#include <limits>
+#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <tuple>
+#include <utility>
#include <vector>
#include "absl/types/span.h"
@@ -164,8 +168,12 @@
// Returns the header from the log file.
const LogFileHeader *log_file_header() const {
- return flatbuffers::GetSizePrefixedRoot<LogFileHeader>(
- configuration_.data());
+ return &raw_log_file_header_.message();
+ }
+
+ // Returns the raw data of the header from the log file.
+ const FlatbufferVector<LogFileHeader> &raw_log_file_header() const {
+ return raw_log_file_header_;
}
// Returns the minimum maount of data needed to queue up for sorting before
@@ -191,8 +199,8 @@
// Log chunk reader.
SpanReader span_reader_;
- // Vector holding the data for the configuration.
- std::vector<uint8_t> configuration_;
+ // Vector holding the raw data for the log file header.
+ FlatbufferVector<LogFileHeader> raw_log_file_header_;
// Minimum amount of data to queue up for sorting before we are guarenteed
// to not see data out of order.
@@ -259,6 +267,10 @@
return &log_file_header_.message();
}
+ const FlatbufferVector<LogFileHeader> &raw_log_file_header() const {
+ return log_file_header_;
+ }
+
// Returns the starting time for this set of log files.
monotonic_clock::time_point monotonic_start_time() {
return monotonic_clock::time_point(
@@ -331,7 +343,7 @@
size_t next_filename_index_ = 0;
// Log file header to report. This is a copy.
- FlatbufferDetachedBuffer<LogFileHeader> log_file_header_;
+ FlatbufferVector<LogFileHeader> log_file_header_;
// Current log file being read.
std::unique_ptr<MessageReader> message_reader_;
@@ -617,7 +629,7 @@
std::vector<std::unique_ptr<SplitMessageReader>> split_message_readers_;
// The log header we are claiming to be.
- FlatbufferDetachedBuffer<LogFileHeader> log_file_header_;
+ FlatbufferVector<LogFileHeader> log_file_header_;
// The timestamp mergers which combine data from the split message readers.
std::vector<TimestampMerger> timestamp_mergers_;
diff --git a/aos/flatbuffer_merge.h b/aos/flatbuffer_merge.h
index 418ea5e..5535763 100644
--- a/aos/flatbuffer_merge.h
+++ b/aos/flatbuffer_merge.h
@@ -89,6 +89,12 @@
reinterpret_cast<const flatbuffers::Table *>(t2));
}
+template <class T>
+inline bool CompareFlatBuffer(const aos::Flatbuffer<T> &t1,
+ const aos::Flatbuffer<T> &t2) {
+ return t1.span() == t2.span();
+}
+
} // namespace aos
#endif // AOS_FLATBUFFER_MERGE_H_