blob: ca36ace00c9798a0de386a06e066608bfbcb18ec [file] [log] [blame]
Austin Schuha36c8902019-12-30 18:07:15 -08001#ifndef AOS_EVENTS_LOGGING_LOGFILE_UTILS_H_
2#define AOS_EVENTS_LOGGING_LOGFILE_UTILS_H_
3
4#include <sys/uio.h>
5
Austin Schuh97789fc2020-08-01 14:42:45 -07006#include <chrono>
Austin Schuh05b70472020-01-01 17:11:17 -08007#include <deque>
Austin Schuh97789fc2020-08-01 14:42:45 -07008#include <limits>
9#include <memory>
Austin Schuh05b70472020-01-01 17:11:17 -080010#include <optional>
Austin Schuhfa895892020-01-07 20:07:41 -080011#include <string>
Austin Schuha36c8902019-12-30 18:07:15 -080012#include <string_view>
Brian Silverman98360e22020-04-28 16:51:20 -070013#include <tuple>
Austin Schuh97789fc2020-08-01 14:42:45 -070014#include <utility>
Austin Schuha36c8902019-12-30 18:07:15 -080015#include <vector>
16
Austin Schuh4b5c22a2020-11-30 22:58:43 -080017#include "absl/container/btree_set.h"
Austin Schuh05b70472020-01-01 17:11:17 -080018#include "absl/types/span.h"
Brian Silvermanf51499a2020-09-21 12:49:08 -070019#include "aos/containers/resizeable_buffer.h"
Austin Schuha36c8902019-12-30 18:07:15 -080020#include "aos/events/event_loop.h"
Austin Schuh2dc8c7d2021-07-01 17:41:28 -070021#include "aos/events/logging/boot_timestamp.h"
Brian Silvermanf51499a2020-09-21 12:49:08 -070022#include "aos/events/logging/buffer_encoder.h"
Austin Schuhc41603c2020-10-11 16:17:37 -070023#include "aos/events/logging/logfile_sorting.h"
Austin Schuha36c8902019-12-30 18:07:15 -080024#include "aos/events/logging/logger_generated.h"
Brian Silvermanf51499a2020-09-21 12:49:08 -070025#include "aos/flatbuffers.h"
Austin Schuhf2d0e682022-10-16 14:20:58 -070026#include "aos/network/remote_message_generated.h"
Austin Schuha36c8902019-12-30 18:07:15 -080027#include "flatbuffers/flatbuffers.h"
28
Brian Silvermanf51499a2020-09-21 12:49:08 -070029namespace aos::logger {
Austin Schuha36c8902019-12-30 18:07:15 -080030
31enum class LogType : uint8_t {
32 // The message originated on this node and should be logged here.
33 kLogMessage,
34 // The message originated on another node, but only the delivery times are
35 // logged here.
36 kLogDeliveryTimeOnly,
37 // The message originated on another node. Log it and the delivery times
38 // together. The message_gateway is responsible for logging any messages
39 // which didn't get delivered.
Austin Schuh6f3babe2020-01-26 20:34:50 -080040 kLogMessageAndDeliveryTime,
41 // The message originated on the other node and should be logged on this node.
42 kLogRemoteMessage
Austin Schuha36c8902019-12-30 18:07:15 -080043};
44
Austin Schuha36c8902019-12-30 18:07:15 -080045// This class manages efficiently writing a sequence of detached buffers to a
Brian Silvermanf51499a2020-09-21 12:49:08 -070046// file. It encodes them, queues them up, and batches the write operation.
Austin Schuha36c8902019-12-30 18:07:15 -080047class DetachedBufferWriter {
48 public:
Brian Silvermana9f2ec92020-10-06 18:00:53 -070049 // Marker struct for one of our constructor overloads.
50 struct already_out_of_space_t {};
51
Brian Silvermanf51499a2020-09-21 12:49:08 -070052 DetachedBufferWriter(std::string_view filename,
Austin Schuh48d10d62022-10-16 22:19:23 -070053 std::unique_ptr<DataEncoder> encoder);
Brian Silvermana9f2ec92020-10-06 18:00:53 -070054 // Creates a dummy instance which won't even open a file. It will act as if
55 // opening the file ran out of space immediately.
56 DetachedBufferWriter(already_out_of_space_t) : ran_out_of_space_(true) {}
Austin Schuh2f8fd752020-09-01 22:38:28 -070057 DetachedBufferWriter(DetachedBufferWriter &&other);
58 DetachedBufferWriter(const DetachedBufferWriter &) = delete;
59
Austin Schuha36c8902019-12-30 18:07:15 -080060 ~DetachedBufferWriter();
61
Austin Schuh2f8fd752020-09-01 22:38:28 -070062 DetachedBufferWriter &operator=(DetachedBufferWriter &&other);
Brian Silverman98360e22020-04-28 16:51:20 -070063 DetachedBufferWriter &operator=(const DetachedBufferWriter &) = delete;
64
Austin Schuh6f3babe2020-01-26 20:34:50 -080065 std::string_view filename() const { return filename_; }
66
Brian Silvermana9f2ec92020-10-06 18:00:53 -070067 // This will be true until Close() is called, unless the file couldn't be
68 // created due to running out of space.
69 bool is_open() const { return fd_ != -1; }
70
Brian Silvermanf51499a2020-09-21 12:49:08 -070071 // Queues up a finished FlatBufferBuilder to be encoded and written.
72 //
73 // Triggers a flush if there's enough data queued up.
74 //
75 // Steals the detached buffer from it.
Austin Schuh48d10d62022-10-16 22:19:23 -070076 void CopyMessage(DataEncoder::Copier *coppier,
77 aos::monotonic_clock::time_point now);
Austin Schuha36c8902019-12-30 18:07:15 -080078
Brian Silverman0465fcf2020-09-24 00:29:18 -070079 // Indicates we got ENOSPC when trying to write. After this returns true, no
80 // further data is written.
81 bool ran_out_of_space() const { return ran_out_of_space_; }
82
83 // To avoid silently failing to write logfiles, you must call this before
84 // destruction if ran_out_of_space() is true and the situation has been
85 // handled.
86 void acknowledge_out_of_space() {
87 CHECK(ran_out_of_space_);
88 acknowledge_ran_out_of_space_ = true;
89 }
90
91 // Fully flushes and closes the underlying file now. No additional data may be
92 // enqueued after calling this.
93 //
94 // This will be performed in the destructor automatically.
95 //
96 // Note that this may set ran_out_of_space().
97 void Close();
98
Brian Silvermanf51499a2020-09-21 12:49:08 -070099 // Returns the total number of bytes written and currently queued.
Austin Schuha426f1f2021-03-31 22:27:41 -0700100 size_t total_bytes() const {
101 if (!encoder_) {
102 return 0;
103 }
104 return encoder_->total_bytes();
105 }
Austin Schuha36c8902019-12-30 18:07:15 -0800106
Brian Silvermanf51499a2020-09-21 12:49:08 -0700107 // The maximum time for a single write call, or 0 if none have been performed.
108 std::chrono::nanoseconds max_write_time() const { return max_write_time_; }
109 // The number of bytes in the longest write call, or -1 if none have been
110 // performed.
111 int max_write_time_bytes() const { return max_write_time_bytes_; }
112 // The number of buffers in the longest write call, or -1 if none have been
113 // performed.
114 int max_write_time_messages() const { return max_write_time_messages_; }
115 // The total time spent in write calls.
116 std::chrono::nanoseconds total_write_time() const {
117 return total_write_time_;
118 }
119 // The total number of writes which have been performed.
120 int total_write_count() const { return total_write_count_; }
121 // The total number of messages which have been written.
122 int total_write_messages() const { return total_write_messages_; }
123 // The total number of bytes which have been written.
124 int total_write_bytes() const { return total_write_bytes_; }
125 void ResetStatistics() {
126 max_write_time_ = std::chrono::nanoseconds::zero();
127 max_write_time_bytes_ = -1;
128 max_write_time_messages_ = -1;
129 total_write_time_ = std::chrono::nanoseconds::zero();
130 total_write_count_ = 0;
131 total_write_messages_ = 0;
132 total_write_bytes_ = 0;
133 }
Brian Silverman98360e22020-04-28 16:51:20 -0700134
Austin Schuha36c8902019-12-30 18:07:15 -0800135 private:
Brian Silvermanf51499a2020-09-21 12:49:08 -0700136 // Performs a single writev call with as much of the data we have queued up as
137 // possible.
138 //
139 // This will normally take all of the data we have queued up, unless an
140 // encoder has spit out a big enough chunk all at once that we can't manage
141 // all of it.
142 void Flush();
143
Brian Silverman0465fcf2020-09-24 00:29:18 -0700144 // write_return is what write(2) or writev(2) returned. write_size is the
145 // number of bytes we expected it to write.
146 void HandleWriteReturn(ssize_t write_return, size_t write_size);
147
Brian Silvermanf51499a2020-09-21 12:49:08 -0700148 void UpdateStatsForWrite(aos::monotonic_clock::duration duration,
149 ssize_t written, int iovec_size);
150
151 // Flushes data if we've reached the threshold to do that as part of normal
Austin Schuhbd06ae42021-03-31 22:48:21 -0700152 // operation either due to the outstanding queued data, or because we have
153 // passed our flush period. now is the current time to save some CPU grabbing
154 // the current time. It just needs to be close.
155 void FlushAtThreshold(aos::monotonic_clock::time_point now);
Brian Silvermanf51499a2020-09-21 12:49:08 -0700156
Austin Schuh2f8fd752020-09-01 22:38:28 -0700157 std::string filename_;
Austin Schuh48d10d62022-10-16 22:19:23 -0700158 std::unique_ptr<DataEncoder> encoder_;
Austin Schuh6f3babe2020-01-26 20:34:50 -0800159
Austin Schuha36c8902019-12-30 18:07:15 -0800160 int fd_ = -1;
Brian Silverman0465fcf2020-09-24 00:29:18 -0700161 bool ran_out_of_space_ = false;
162 bool acknowledge_ran_out_of_space_ = false;
Austin Schuha36c8902019-12-30 18:07:15 -0800163
Austin Schuha36c8902019-12-30 18:07:15 -0800164 // List of iovecs to use with writev. This is a member variable to avoid
165 // churn.
166 std::vector<struct iovec> iovec_;
Brian Silvermanf51499a2020-09-21 12:49:08 -0700167
168 std::chrono::nanoseconds max_write_time_ = std::chrono::nanoseconds::zero();
169 int max_write_time_bytes_ = -1;
170 int max_write_time_messages_ = -1;
171 std::chrono::nanoseconds total_write_time_ = std::chrono::nanoseconds::zero();
172 int total_write_count_ = 0;
173 int total_write_messages_ = 0;
174 int total_write_bytes_ = 0;
Austin Schuhbd06ae42021-03-31 22:48:21 -0700175
176 aos::monotonic_clock::time_point last_flush_time_ =
177 aos::monotonic_clock::min_time;
Austin Schuha36c8902019-12-30 18:07:15 -0800178};
179
Austin Schuhf2d0e682022-10-16 14:20:58 -0700180// Repacks the provided RemoteMessage into fbb.
181flatbuffers::Offset<MessageHeader> PackRemoteMessage(
182 flatbuffers::FlatBufferBuilder *fbb,
183 const message_bridge::RemoteMessage *msg, int channel_index,
184 const aos::monotonic_clock::time_point monotonic_timestamp_time);
185
186constexpr flatbuffers::uoffset_t PackRemoteMessageSize() { return 96u; }
187size_t PackRemoteMessageInline(
188 uint8_t *data, const message_bridge::RemoteMessage *msg, int channel_index,
189 const aos::monotonic_clock::time_point monotonic_timestamp_time);
190
Austin Schuha36c8902019-12-30 18:07:15 -0800191// Packes a message pointed to by the context into a MessageHeader.
192flatbuffers::Offset<MessageHeader> PackMessage(
193 flatbuffers::FlatBufferBuilder *fbb, const Context &context,
194 int channel_index, LogType log_type);
195
Austin Schuhfa30c352022-10-16 11:12:02 -0700196// Returns the size that the packed message from PackMessage or
197// PackMessageInline will be.
Austin Schuh48d10d62022-10-16 22:19:23 -0700198flatbuffers::uoffset_t PackMessageSize(LogType log_type, size_t data_size);
Austin Schuhfa30c352022-10-16 11:12:02 -0700199
200// Packs the provided message pointed to by context into the provided buffer.
201// This is equivalent to PackMessage, but doesn't require allocating a
202// FlatBufferBuilder underneath.
203size_t PackMessageInline(uint8_t *data, const Context &contex,
204 int channel_index, LogType log_type);
205
Austin Schuh05b70472020-01-01 17:11:17 -0800206// Class to read chunks out of a log file.
207class SpanReader {
208 public:
Austin Schuhcd368422021-11-22 21:23:29 -0800209 SpanReader(std::string_view filename, bool quiet = false);
Austin Schuha36c8902019-12-30 18:07:15 -0800210
Austin Schuh6f3babe2020-01-26 20:34:50 -0800211 std::string_view filename() const { return filename_; }
212
Brian Smarttea913d42021-12-10 15:02:38 -0800213 size_t TotalRead() const { return total_read_; }
214 size_t TotalConsumed() const { return total_consumed_; }
Austin Schuh60e77942022-05-16 17:48:24 -0700215 bool IsIncomplete() const {
216 return is_finished_ && total_consumed_ < total_read_;
217 }
Brian Smarttea913d42021-12-10 15:02:38 -0800218
Austin Schuhcf5f6442021-07-06 10:43:28 -0700219 // Returns a span with the data for the next message from the log file,
220 // including the size. The result is only guarenteed to be valid until
221 // ReadMessage() or PeekMessage() is called again.
Austin Schuh05b70472020-01-01 17:11:17 -0800222 absl::Span<const uint8_t> ReadMessage();
223
Austin Schuhcf5f6442021-07-06 10:43:28 -0700224 // Returns a span with the data for the next message without consuming it.
225 // Multiple calls to PeekMessage return the same data. ReadMessage or
226 // ConsumeMessage must be called to get the next message.
227 absl::Span<const uint8_t> PeekMessage();
228 // Consumes the message so the next call to ReadMessage or PeekMessage returns
229 // new data. This does not invalidate the data.
230 void ConsumeMessage();
231
Austin Schuh05b70472020-01-01 17:11:17 -0800232 private:
233 // TODO(austin): Optimization:
234 // Allocate the 256k blocks like we do today. But, refcount them with
235 // shared_ptr pointed to by the messageheader that is returned. This avoids
236 // the copy. Need to do more benchmarking.
Brian Silvermanf51499a2020-09-21 12:49:08 -0700237 // And (Brian): Consider just mmapping the file and handing out refcounted
238 // pointers into that too.
Austin Schuh05b70472020-01-01 17:11:17 -0800239
240 // Reads a chunk of data into data_. Returns false if no data was read.
241 bool ReadBlock();
242
Austin Schuhc41603c2020-10-11 16:17:37 -0700243 std::string filename_;
Austin Schuh6f3babe2020-01-26 20:34:50 -0800244
Brian Silvermanf51499a2020-09-21 12:49:08 -0700245 // File reader and data decoder.
246 std::unique_ptr<DataDecoder> decoder_;
Austin Schuh05b70472020-01-01 17:11:17 -0800247
Brian Silvermanf51499a2020-09-21 12:49:08 -0700248 // Vector to read into.
249 ResizeableBuffer data_;
Austin Schuh05b70472020-01-01 17:11:17 -0800250
251 // Amount of data consumed already in data_.
252 size_t consumed_data_ = 0;
Brian Smarttea913d42021-12-10 15:02:38 -0800253
254 // Accumulates the total volume of bytes read from filename_
255 size_t total_read_ = 0;
256
257 // Accumulates the total volume of read bytes that were 'consumed' into
258 // messages. May be less than total_read_, if the last message (span) is
259 // either truncated or somehow corrupt.
260 size_t total_consumed_ = 0;
261
262 // Reached the end, no more readable messages.
263 bool is_finished_ = false;
Austin Schuh05b70472020-01-01 17:11:17 -0800264};
265
Brian Silvermanfee16972021-09-14 12:06:38 -0700266// Reads the last header from a log file. This handles any duplicate headers
267// that were written.
268std::optional<SizePrefixedFlatbufferVector<LogFileHeader>> ReadHeader(
269 SpanReader *span_reader);
270std::optional<SizePrefixedFlatbufferVector<LogFileHeader>> ReadHeader(
271 std::string_view filename);
272// Reads the Nth message from a log file, excluding the header. Note: this
273// doesn't handle duplicate headers.
274std::optional<SizePrefixedFlatbufferVector<MessageHeader>> ReadNthMessage(
275 std::string_view filename, size_t n);
276
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700277class UnpackedMessageHeader;
278
Austin Schuh05b70472020-01-01 17:11:17 -0800279// Class which handles reading the header and messages from the log file. This
280// handles any per-file state left before merging below.
281class MessageReader {
282 public:
283 MessageReader(std::string_view filename);
284
Austin Schuh6f3babe2020-01-26 20:34:50 -0800285 std::string_view filename() const { return span_reader_.filename(); }
286
Austin Schuh05b70472020-01-01 17:11:17 -0800287 // Returns the header from the log file.
288 const LogFileHeader *log_file_header() const {
Austin Schuh97789fc2020-08-01 14:42:45 -0700289 return &raw_log_file_header_.message();
290 }
291
292 // Returns the raw data of the header from the log file.
Austin Schuhadd6eb32020-11-09 21:24:26 -0800293 const SizePrefixedFlatbufferVector<LogFileHeader> &raw_log_file_header()
294 const {
Austin Schuh97789fc2020-08-01 14:42:45 -0700295 return raw_log_file_header_;
Austin Schuh05b70472020-01-01 17:11:17 -0800296 }
297
298 // Returns the minimum maount of data needed to queue up for sorting before
299 // ware guarenteed to not see data out of order.
300 std::chrono::nanoseconds max_out_of_order_duration() const {
301 return max_out_of_order_duration_;
302 }
303
Austin Schuhcde938c2020-02-02 17:30:07 -0800304 // Returns the newest timestamp read out of the log file.
Austin Schuh05b70472020-01-01 17:11:17 -0800305 monotonic_clock::time_point newest_timestamp() const {
306 return newest_timestamp_;
307 }
308
309 // Returns the next message if there is one.
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700310 std::shared_ptr<UnpackedMessageHeader> ReadMessage();
Austin Schuh05b70472020-01-01 17:11:17 -0800311
312 // The time at which we need to read another chunk from the logfile.
313 monotonic_clock::time_point queue_data_time() const {
314 return newest_timestamp() - max_out_of_order_duration();
315 }
316
Brian Smarttea913d42021-12-10 15:02:38 -0800317 // Flag value setters for testing
318 void set_crash_on_corrupt_message_flag(bool b) {
319 crash_on_corrupt_message_flag_ = b;
320 }
321 void set_ignore_corrupt_messages_flag(bool b) {
322 ignore_corrupt_messages_flag_ = b;
323 }
324
Austin Schuh05b70472020-01-01 17:11:17 -0800325 private:
326 // Log chunk reader.
327 SpanReader span_reader_;
328
Austin Schuh97789fc2020-08-01 14:42:45 -0700329 // Vector holding the raw data for the log file header.
Austin Schuhadd6eb32020-11-09 21:24:26 -0800330 SizePrefixedFlatbufferVector<LogFileHeader> raw_log_file_header_;
Austin Schuh05b70472020-01-01 17:11:17 -0800331
332 // Minimum amount of data to queue up for sorting before we are guarenteed
333 // to not see data out of order.
334 std::chrono::nanoseconds max_out_of_order_duration_;
335
336 // Timestamp of the newest message in a channel queue.
337 monotonic_clock::time_point newest_timestamp_ = monotonic_clock::min_time;
Brian Smarttea913d42021-12-10 15:02:38 -0800338
339 // Total volume of verifiable messages from the beginning of the file.
340 // TODO - are message counts also useful?
341 size_t total_verified_before_ = 0;
342
343 // Total volume of messages with corrupted flatbuffer formatting, if any.
344 // Excludes corrupted message content.
345 // TODO - if the layout included something as simple as a CRC (relatively
346 // fast and robust enough) for each span, then corrupted content could be
347 // included in this check.
348 size_t total_corrupted_ = 0;
349
350 // Total volume of verifiable messages intermixed with corrupted messages,
351 // if any. Will be == 0 if total_corrupted_ == 0.
352 size_t total_verified_during_ = 0;
353
354 // Total volume of verifiable messages found after the last corrupted one,
355 // if any. Will be == 0 if total_corrupted_ == 0.
356 size_t total_verified_after_ = 0;
357
358 bool is_corrupted() const { return total_corrupted_ > 0; }
359
360 bool crash_on_corrupt_message_flag_ = true;
361 bool ignore_corrupt_messages_flag_ = false;
Austin Schuh05b70472020-01-01 17:11:17 -0800362};
363
Austin Schuhc41603c2020-10-11 16:17:37 -0700364// A class to seamlessly read messages from a list of part files.
365class PartsMessageReader {
366 public:
367 PartsMessageReader(LogParts log_parts);
368
369 std::string_view filename() const { return message_reader_.filename(); }
370
Austin Schuhd2f96102020-12-01 20:27:29 -0800371 // Returns the LogParts that holds the filenames we are reading.
372 const LogParts &parts() const { return parts_; }
373
Austin Schuh4b5c22a2020-11-30 22:58:43 -0800374 const LogFileHeader *log_file_header() const {
375 return message_reader_.log_file_header();
376 }
377
Austin Schuhc41603c2020-10-11 16:17:37 -0700378 // Returns the minimum amount of data needed to queue up for sorting before
379 // we are guarenteed to not see data out of order.
380 std::chrono::nanoseconds max_out_of_order_duration() const {
381 return message_reader_.max_out_of_order_duration();
382 }
383
384 // Returns the newest timestamp read out of the log file.
385 monotonic_clock::time_point newest_timestamp() const {
386 return newest_timestamp_;
387 }
388
389 // Returns the next message if there is one, or nullopt if we have reached the
390 // end of all the files.
391 // Note: reading the next message may change the max_out_of_order_duration().
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700392 std::shared_ptr<UnpackedMessageHeader> ReadMessage();
Austin Schuhc41603c2020-10-11 16:17:37 -0700393
Austin Schuh48507722021-07-17 17:29:24 -0700394 // Returns the boot count for the requested node, or std::nullopt if we don't
395 // know.
396 std::optional<size_t> boot_count(size_t node_index) const {
397 CHECK_GE(node_index, 0u);
398 CHECK_LT(node_index, boot_counts_.size());
399 return boot_counts_[node_index];
400 }
401
Austin Schuhc41603c2020-10-11 16:17:37 -0700402 private:
403 // Opens the next log and updates message_reader_. Sets done_ if there is
404 // nothing more to do.
405 void NextLog();
Austin Schuh48507722021-07-17 17:29:24 -0700406 void ComputeBootCounts();
Austin Schuhc41603c2020-10-11 16:17:37 -0700407
408 const LogParts parts_;
409 size_t next_part_index_ = 1u;
410 bool done_ = false;
411 MessageReader message_reader_;
Brian Silvermanfee16972021-09-14 12:06:38 -0700412 // We instantiate the next one early, to allow implementations to prefetch.
413 // TODO(Brian): To get optimal performance when downloading, this needs more
414 // communication with the implementation to prioritize the next part and add
415 // more parallelism when it helps. Maybe some kind of a queue of parts in
416 // order, and the implementation gets to pull however many make sense off the
417 // front?
418 std::optional<MessageReader> next_message_reader_;
Austin Schuhc41603c2020-10-11 16:17:37 -0700419
Austin Schuh315b96b2020-12-11 21:21:12 -0800420 // True after we have seen a message after the start of the log. The
421 // guarentees on logging essentially are that all data from before the
422 // starting time of the log may be arbitrarily out of order, but once we get
423 // max_out_of_order_duration past the start, everything will remain within
424 // max_out_of_order_duration. We shouldn't see anything before the start
425 // after we've seen a message that is at least max_out_of_order_duration after
426 // the start.
427 bool after_start_ = false;
428
Austin Schuhc41603c2020-10-11 16:17:37 -0700429 monotonic_clock::time_point newest_timestamp_ = monotonic_clock::min_time;
Austin Schuh48507722021-07-17 17:29:24 -0700430
431 // Per node boot counts.
432 std::vector<std::optional<size_t>> boot_counts_;
Austin Schuhc41603c2020-10-11 16:17:37 -0700433};
434
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700435// Stores MessageHeader as a flat header and inline, aligned block of data.
436class UnpackedMessageHeader {
437 public:
James Kuszmaul9776b392023-01-14 14:08:08 -0800438 UnpackedMessageHeader(
439 uint32_t channel_index, monotonic_clock::time_point monotonic_sent_time,
440 realtime_clock::time_point realtime_sent_time, uint32_t queue_index,
441 std::optional<monotonic_clock::time_point> monotonic_remote_time,
442 std::optional<realtime_clock::time_point> realtime_remote_time,
443 std::optional<uint32_t> remote_queue_index,
444 monotonic_clock::time_point monotonic_timestamp_time,
445 bool has_monotonic_timestamp_time, absl::Span<const uint8_t> span)
446 : channel_index(channel_index),
447 monotonic_sent_time(monotonic_sent_time),
448 realtime_sent_time(realtime_sent_time),
449 queue_index(queue_index),
450 monotonic_remote_time(monotonic_remote_time),
451 realtime_remote_time(realtime_remote_time),
452 remote_queue_index(remote_queue_index),
453 monotonic_timestamp_time(monotonic_timestamp_time),
454 has_monotonic_timestamp_time(has_monotonic_timestamp_time),
455 span(span) {}
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700456 UnpackedMessageHeader(const UnpackedMessageHeader &) = delete;
457 UnpackedMessageHeader &operator=(const UnpackedMessageHeader &) = delete;
458
459 // The channel.
460 uint32_t channel_index = 0xffffffff;
461
462 monotonic_clock::time_point monotonic_sent_time;
463 realtime_clock::time_point realtime_sent_time;
464
465 // The local queue index.
466 uint32_t queue_index = 0xffffffff;
467
Austin Schuh826e6ce2021-11-18 20:33:10 -0800468 std::optional<aos::monotonic_clock::time_point> monotonic_remote_time;
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700469
470 std::optional<realtime_clock::time_point> realtime_remote_time;
471 std::optional<uint32_t> remote_queue_index;
472
473 // This field is defaulted in the flatbuffer, so we need to store both the
474 // possibly defaulted value and whether it is defaulted.
475 monotonic_clock::time_point monotonic_timestamp_time;
476 bool has_monotonic_timestamp_time;
477
478 static std::shared_ptr<UnpackedMessageHeader> MakeMessage(
479 const MessageHeader &message);
480
481 // Note: we are storing a span here because we need something to put in the
482 // SharedSpan pointer that RawSender takes. We are using the aliasing
483 // constructor of shared_ptr to avoid the allocation, and it needs a nice
484 // pointer to track.
485 absl::Span<const uint8_t> span;
486
487 char actual_data[];
488
489 private:
490 ~UnpackedMessageHeader() {}
491
492 static void DestroyAndFree(UnpackedMessageHeader *p) {
493 p->~UnpackedMessageHeader();
494 free(p);
495 }
496};
497
498std::ostream &operator<<(std::ostream &os,
499 const UnpackedMessageHeader &message);
500
Austin Schuh1be0ce42020-11-29 22:43:26 -0800501// Struct to hold a message as it gets sorted on a single node.
502struct Message {
503 // The channel.
504 uint32_t channel_index = 0xffffffff;
505 // The local queue index.
Austin Schuh58646e22021-08-23 23:51:46 -0700506 // TODO(austin): Technically the boot inside queue_index is redundant with
507 // timestamp. In practice, it is less error-prone to duplicate it. Maybe a
508 // function to return the combined struct?
509 BootQueueIndex queue_index;
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700510 // The local timestamp.
511 BootTimestamp timestamp;
Austin Schuhf16ef6a2021-06-30 21:48:17 -0700512
Austin Schuh48507722021-07-17 17:29:24 -0700513 // Remote boot when this is a timestamp.
514 size_t monotonic_remote_boot = 0xffffff;
515
516 size_t monotonic_timestamp_boot = 0xffffff;
517
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700518 std::shared_ptr<UnpackedMessageHeader> data;
Austin Schuh1be0ce42020-11-29 22:43:26 -0800519
520 bool operator<(const Message &m2) const;
521 bool operator>=(const Message &m2) const;
Austin Schuh8f52ed52020-11-30 23:12:39 -0800522 bool operator==(const Message &m2) const;
Austin Schuh1be0ce42020-11-29 22:43:26 -0800523};
524
525std::ostream &operator<<(std::ostream &os, const Message &m);
526
Austin Schuhd2f96102020-12-01 20:27:29 -0800527// Structure to hold a full message and all the timestamps, which may or may not
528// have been sent from a remote node. The remote_queue_index will be invalid if
529// this message is from the point of view of the node which sent it.
530struct TimestampedMessage {
531 uint32_t channel_index = 0xffffffff;
532
Austin Schuh58646e22021-08-23 23:51:46 -0700533 BootQueueIndex queue_index;
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700534 BootTimestamp monotonic_event_time;
Austin Schuhd2f96102020-12-01 20:27:29 -0800535 realtime_clock::time_point realtime_event_time = realtime_clock::min_time;
536
Austin Schuh58646e22021-08-23 23:51:46 -0700537 BootQueueIndex remote_queue_index;
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700538 BootTimestamp monotonic_remote_time;
Austin Schuhd2f96102020-12-01 20:27:29 -0800539 realtime_clock::time_point realtime_remote_time = realtime_clock::min_time;
540
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700541 BootTimestamp monotonic_timestamp_time;
Austin Schuh8bf1e632021-01-02 22:41:04 -0800542
Tyler Chatowb7c6eba2021-07-28 14:43:23 -0700543 std::shared_ptr<UnpackedMessageHeader> data;
Austin Schuhd2f96102020-12-01 20:27:29 -0800544};
545
546std::ostream &operator<<(std::ostream &os, const TimestampedMessage &m);
547
Austin Schuh4b5c22a2020-11-30 22:58:43 -0800548// Class to sort the resulting messages from a PartsMessageReader.
549class LogPartsSorter {
550 public:
551 LogPartsSorter(LogParts log_parts);
552
Austin Schuh0ca51f32020-12-25 21:51:45 -0800553 // Returns the parts that this is sorting messages from.
554 const LogParts &parts() const { return parts_message_reader_.parts(); }
Austin Schuh4b5c22a2020-11-30 22:58:43 -0800555
Austin Schuhd2f96102020-12-01 20:27:29 -0800556 monotonic_clock::time_point monotonic_start_time() const {
Austin Schuh0ca51f32020-12-25 21:51:45 -0800557 return parts().monotonic_start_time;
Austin Schuhd2f96102020-12-01 20:27:29 -0800558 }
559 realtime_clock::time_point realtime_start_time() const {
Austin Schuh0ca51f32020-12-25 21:51:45 -0800560 return parts().realtime_start_time;
Austin Schuhd2f96102020-12-01 20:27:29 -0800561 }
562
Austin Schuh4b5c22a2020-11-30 22:58:43 -0800563 // The time this data is sorted until.
564 monotonic_clock::time_point sorted_until() const { return sorted_until_; }
565
566 // Returns the next sorted message from the log file. It is safe to call
567 // std::move() on the result to move the data flatbuffer from it.
568 Message *Front();
569 // Pops the front message. This should only be called after a call to
570 // Front().
571 void PopFront();
572
573 // Returns a debug string representing the contents of this sorter.
574 std::string DebugString() const;
575
576 private:
577 // Log parts reader we are wrapping.
578 PartsMessageReader parts_message_reader_;
579 // Cache of the time we are sorted until.
580 aos::monotonic_clock::time_point sorted_until_ = monotonic_clock::min_time;
581
Austin Schuhb000de62020-12-03 22:00:40 -0800582 // Timestamp of the last message returned. Used to make sure nothing goes
583 // backwards.
584 monotonic_clock::time_point last_message_time_ = monotonic_clock::min_time;
585
Austin Schuh4b5c22a2020-11-30 22:58:43 -0800586 // Set used for efficient sorting of messages. We can benchmark and evaluate
587 // other data structures if this proves to be the bottleneck.
588 absl::btree_set<Message> messages_;
Austin Schuh48507722021-07-17 17:29:24 -0700589
590 // Mapping from channel to source node.
591 // TODO(austin): Should we put this in Boots so it can be cached for everyone?
592 std::vector<size_t> source_node_index_;
Austin Schuh4b5c22a2020-11-30 22:58:43 -0800593};
594
Austin Schuh8f52ed52020-11-30 23:12:39 -0800595// Class to run merge sort on the messages from multiple LogPartsSorter
596// instances.
597class NodeMerger {
598 public:
Austin Schuhd2f96102020-12-01 20:27:29 -0800599 NodeMerger(std::vector<LogParts> parts);
600
Austin Schuhf16ef6a2021-06-30 21:48:17 -0700601 // Copying and moving will mess up the internal raw pointers. Just don't do
602 // it.
603 NodeMerger(NodeMerger const &) = delete;
604 NodeMerger(NodeMerger &&) = delete;
605 void operator=(NodeMerger const &) = delete;
606 void operator=(NodeMerger &&) = delete;
607
Austin Schuhd2f96102020-12-01 20:27:29 -0800608 // Node index in the configuration of this node.
609 int node() const { return node_; }
Austin Schuh8f52ed52020-11-30 23:12:39 -0800610
Austin Schuh0ca51f32020-12-25 21:51:45 -0800611 // List of parts being sorted together.
612 std::vector<const LogParts *> Parts() const;
613
614 const Configuration *configuration() const {
615 return parts_sorters_[0].parts().config.get();
Austin Schuhd2f96102020-12-01 20:27:29 -0800616 }
617
618 monotonic_clock::time_point monotonic_start_time() const {
619 return monotonic_start_time_;
620 }
621 realtime_clock::time_point realtime_start_time() const {
622 return realtime_start_time_;
Austin Schuh8f52ed52020-11-30 23:12:39 -0800623 }
Austin Schuh5dd22842021-11-17 16:09:39 -0800624 monotonic_clock::time_point monotonic_oldest_time() const {
625 return monotonic_oldest_time_;
626 }
Austin Schuh8f52ed52020-11-30 23:12:39 -0800627
628 // The time this data is sorted until.
629 monotonic_clock::time_point sorted_until() const { return sorted_until_; }
630
631 // Returns the next sorted message from the set of log files. It is safe to
632 // call std::move() on the result to move the data flatbuffer from it.
633 Message *Front();
634 // Pops the front message. This should only be called after a call to
635 // Front().
636 void PopFront();
637
638 private:
639 // Unsorted list of all parts sorters.
Austin Schuhd2f96102020-12-01 20:27:29 -0800640 std::vector<LogPartsSorter> parts_sorters_;
Austin Schuh8f52ed52020-11-30 23:12:39 -0800641 // Pointer to the parts sorter holding the current Front message if one
642 // exists, or nullptr if a new one needs to be found.
643 LogPartsSorter *current_ = nullptr;
644 // Cached sorted_until value.
645 aos::monotonic_clock::time_point sorted_until_ = monotonic_clock::min_time;
Austin Schuhd2f96102020-12-01 20:27:29 -0800646
647 // Cached node.
648 int node_;
649
Austin Schuhb000de62020-12-03 22:00:40 -0800650 // Timestamp of the last message returned. Used to make sure nothing goes
651 // backwards.
652 monotonic_clock::time_point last_message_time_ = monotonic_clock::min_time;
653
Austin Schuhd2f96102020-12-01 20:27:29 -0800654 realtime_clock::time_point realtime_start_time_ = realtime_clock::max_time;
655 monotonic_clock::time_point monotonic_start_time_ = monotonic_clock::max_time;
Austin Schuh60e77942022-05-16 17:48:24 -0700656 monotonic_clock::time_point monotonic_oldest_time_ =
657 monotonic_clock::max_time;
Austin Schuhd2f96102020-12-01 20:27:29 -0800658};
659
Austin Schuhf16ef6a2021-06-30 21:48:17 -0700660// Class to concatenate multiple boots worth of logs into a single per-node
661// stream.
662class BootMerger {
663 public:
664 BootMerger(std::vector<LogParts> file);
665
666 // Copying and moving will mess up the internal raw pointers. Just don't do
667 // it.
668 BootMerger(BootMerger const &) = delete;
669 BootMerger(BootMerger &&) = delete;
670 void operator=(BootMerger const &) = delete;
671 void operator=(BootMerger &&) = delete;
672
673 // Node index in the configuration of this node.
674 int node() const { return node_mergers_[0]->node(); }
675
676 // List of parts being sorted together.
677 std::vector<const LogParts *> Parts() const;
678
679 const Configuration *configuration() const {
680 return node_mergers_[0]->configuration();
681 }
682
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700683 monotonic_clock::time_point monotonic_start_time(size_t boot) const {
684 CHECK_LT(boot, node_mergers_.size());
685 return node_mergers_[boot]->monotonic_start_time();
Austin Schuhf16ef6a2021-06-30 21:48:17 -0700686 }
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700687 realtime_clock::time_point realtime_start_time(size_t boot) const {
688 CHECK_LT(boot, node_mergers_.size());
689 return node_mergers_[boot]->realtime_start_time();
Austin Schuhf16ef6a2021-06-30 21:48:17 -0700690 }
Austin Schuh5dd22842021-11-17 16:09:39 -0800691 monotonic_clock::time_point monotonic_oldest_time(size_t boot) const {
692 CHECK_LT(boot, node_mergers_.size());
693 return node_mergers_[boot]->monotonic_oldest_time();
694 }
Austin Schuhf16ef6a2021-06-30 21:48:17 -0700695
696 bool started() const {
697 return node_mergers_[index_]->sorted_until() != monotonic_clock::min_time ||
698 index_ != 0;
699 }
700
701 // Returns the next sorted message from the set of log files. It is safe to
702 // call std::move() on the result to move the data flatbuffer from it.
703 Message *Front();
704 // Pops the front message. This should only be called after a call to
705 // Front().
706 void PopFront();
707
708 private:
709 int index_ = 0;
710
711 // TODO(austin): Sanjay points out this is pretty inefficient. Don't keep so
712 // many things open.
713 std::vector<std::unique_ptr<NodeMerger>> node_mergers_;
714};
715
Austin Schuhd2f96102020-12-01 20:27:29 -0800716// Class to match timestamps with the corresponding data from other nodes.
Austin Schuh79b30942021-01-24 22:32:21 -0800717//
718// This class also buffers data for the node it represents, and supports
719// notifying when new data is queued as well as queueing until a point in time.
Austin Schuhd2f96102020-12-01 20:27:29 -0800720class TimestampMapper {
721 public:
722 TimestampMapper(std::vector<LogParts> file);
723
724 // Copying and moving will mess up the internal raw pointers. Just don't do
725 // it.
726 TimestampMapper(TimestampMapper const &) = delete;
727 TimestampMapper(TimestampMapper &&) = delete;
728 void operator=(TimestampMapper const &) = delete;
729 void operator=(TimestampMapper &&) = delete;
730
731 // TODO(austin): It would be super helpful to provide a way to queue up to
732 // time X without matching timestamps, and to then be able to pull the
733 // timestamps out of this queue. This lets us bootstrap time estimation
734 // without exploding memory usage worst case.
735
Austin Schuh0ca51f32020-12-25 21:51:45 -0800736 const Configuration *configuration() const { return configuration_.get(); }
Austin Schuhd2f96102020-12-01 20:27:29 -0800737
738 // Returns which node this is sorting for.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700739 size_t node() const { return boot_merger_.node(); }
Austin Schuhd2f96102020-12-01 20:27:29 -0800740
741 // The start time of this log.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700742 monotonic_clock::time_point monotonic_start_time(size_t boot) const {
743 return boot_merger_.monotonic_start_time(boot);
Austin Schuhd2f96102020-12-01 20:27:29 -0800744 }
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700745 realtime_clock::time_point realtime_start_time(size_t boot) const {
746 return boot_merger_.realtime_start_time(boot);
Austin Schuhd2f96102020-12-01 20:27:29 -0800747 }
Austin Schuh5dd22842021-11-17 16:09:39 -0800748 // Returns the oldest timestamp on a message on this boot.
749 monotonic_clock::time_point monotonic_oldest_time(size_t boot) const {
750 return boot_merger_.monotonic_oldest_time(boot);
751 }
Austin Schuhd2f96102020-12-01 20:27:29 -0800752
753 // Uses timestamp_mapper as the peer for its node. Only one mapper may be set
754 // for each node. Peers are used to look up the data for timestamps on this
755 // node.
756 void AddPeer(TimestampMapper *timestamp_mapper);
757
Austin Schuh24bf4972021-06-29 22:09:08 -0700758 // Returns true if anything has been queued up.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700759 bool started() const { return boot_merger_.started(); }
Austin Schuhd2f96102020-12-01 20:27:29 -0800760
761 // Returns the next message for this node.
762 TimestampedMessage *Front();
763 // Pops the next message. Front must be called first.
764 void PopFront();
765
766 // Returns debug information about this node.
767 std::string DebugString() const;
768
Austin Schuh79b30942021-01-24 22:32:21 -0800769 // Queues data the provided time.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700770 void QueueUntil(BootTimestamp queue_time);
Austin Schuhe639ea12021-01-25 13:00:22 -0800771 // Queues until we have time_estimation_buffer of data in the queue.
772 void QueueFor(std::chrono::nanoseconds time_estimation_buffer);
Austin Schuh79b30942021-01-24 22:32:21 -0800773
Austin Schuh06601222021-01-26 17:02:50 -0800774 // Queues until the condition is met.
775 template <typename T>
776 void QueueUntilCondition(T fn) {
777 while (true) {
778 if (fn()) {
779 break;
780 }
781 if (!QueueMatched()) {
782 break;
783 }
784 }
785 }
786
Eric Schmiedebergb38477e2022-12-02 16:08:04 -0700787 // Sets the callback that can be used to skip messages.
788 void set_replay_channels_callback(
789 std::function<bool(const TimestampedMessage &)> fn) {
790 replay_channels_callback_ = fn;
791 }
792
Austin Schuh79b30942021-01-24 22:32:21 -0800793 // Sets a callback to be called whenever a full message is queued.
794 void set_timestamp_callback(std::function<void(TimestampedMessage *)> fn) {
795 timestamp_callback_ = fn;
796 }
797
Austin Schuhd2f96102020-12-01 20:27:29 -0800798 private:
Eric Schmiedebergb38477e2022-12-02 16:08:04 -0700799 // Result of MaybeQueueMatched
800 enum class MatchResult : uint8_t {
801 kEndOfFile, // End of the log file being read
802 kQueued, // Message was queued
803 kSkipped // Message was skipped over
804 };
805
Austin Schuhd2f96102020-12-01 20:27:29 -0800806 // The state for a remote node. This holds the data that needs to be matched
807 // with the remote node's timestamps.
808 struct NodeData {
809 // True if we should save data here. This should be true if any of the
810 // bools in delivered below are true.
811 bool any_delivered = false;
812
Austin Schuh36c00932021-07-19 18:13:21 -0700813 // True if we have a peer and therefore should be saving data for it.
814 bool save_for_peer = false;
815
Austin Schuhd2f96102020-12-01 20:27:29 -0800816 // Peer pointer. This node is only to be considered if a peer is set.
817 TimestampMapper *peer = nullptr;
818
819 struct ChannelData {
820 // Deque per channel. This contains the data from the outside
821 // TimestampMapper node which is relevant for the node this NodeData
822 // points to.
823 std::deque<Message> messages;
824 // Bool tracking per channel if a message is delivered to the node this
825 // NodeData represents.
826 bool delivered = false;
Austin Schuh6a7358f2021-11-18 22:40:40 -0800827 // The TTL for delivery.
828 std::chrono::nanoseconds time_to_live = std::chrono::nanoseconds(0);
Austin Schuhd2f96102020-12-01 20:27:29 -0800829 };
830
831 // Vector with per channel data.
832 std::vector<ChannelData> channels;
833 };
834
835 // Returns (and forgets about) the data for the provided timestamp message
836 // showing when it was delivered to this node.
837 Message MatchingMessageFor(const Message &message);
838
839 // Queues up a single message into our message queue, and any nodes that this
840 // message is delivered to. Returns true if one was available, false
841 // otherwise.
842 bool Queue();
843
Austin Schuh79b30942021-01-24 22:32:21 -0800844 // Queues up a single matched message into our matched message queue. Returns
845 // true if one was queued, and false otherwise.
846 bool QueueMatched();
847
Eric Schmiedebergb38477e2022-12-02 16:08:04 -0700848 // Queues a message if the replay_channels_callback is passed and the end of
849 // the log file has not been reached.
850 MatchResult MaybeQueueMatched();
851
Austin Schuhd2f96102020-12-01 20:27:29 -0800852 // Queues up data until we have at least one message >= to time t.
853 // Useful for triggering a remote node to read enough data to have the
854 // timestamp you care about available.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700855 void QueueUnmatchedUntil(BootTimestamp t);
Austin Schuhd2f96102020-12-01 20:27:29 -0800856
Austin Schuh79b30942021-01-24 22:32:21 -0800857 // Queues m into matched_messages_.
858 void QueueMessage(Message *m);
Austin Schuhd2f96102020-12-01 20:27:29 -0800859
Eric Schmiedebergb38477e2022-12-02 16:08:04 -0700860 // If a replay_channels_callback was set and the callback returns false, a
861 // matched message is popped and true is returned. Otherwise false is
862 // returned.
863 bool CheckReplayChannelsAndMaybePop(const TimestampedMessage &message);
864
Austin Schuh58646e22021-08-23 23:51:46 -0700865 // Returns the name of the node this class is sorting for.
866 std::string_view node_name() const {
867 return configuration_->has_nodes() ? configuration_->nodes()
868 ->Get(boot_merger_.node())
869 ->name()
870 ->string_view()
871 : "(single node)";
872 }
873
Austin Schuhd2f96102020-12-01 20:27:29 -0800874 // The node merger to source messages from.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700875 BootMerger boot_merger_;
Austin Schuh0ca51f32020-12-25 21:51:45 -0800876
877 std::shared_ptr<const Configuration> configuration_;
878
Austin Schuhd2f96102020-12-01 20:27:29 -0800879 // The buffer of messages for this node. These are not matched with any
880 // remote data.
881 std::deque<Message> messages_;
882 // The node index for the source node for each channel.
883 std::vector<size_t> source_node_;
884
885 // Vector per node. Not all nodes will have anything.
886 std::vector<NodeData> nodes_data_;
887
888 // Latest message to return.
Austin Schuh79b30942021-01-24 22:32:21 -0800889 std::deque<TimestampedMessage> matched_messages_;
Austin Schuhd2f96102020-12-01 20:27:29 -0800890
Austin Schuh79b30942021-01-24 22:32:21 -0800891 // Tracks the state of the first message in matched_messages_. Do we need to
892 // update it, is it valid, or should we return nullptr?
Austin Schuhd2f96102020-12-01 20:27:29 -0800893 enum class FirstMessage {
894 kNeedsUpdate,
895 kInMessage,
896 kNullptr,
897 };
898 FirstMessage first_message_ = FirstMessage::kNeedsUpdate;
899
900 // Timestamp of the last message returned. Used to make sure nothing goes
901 // backwards.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700902 BootTimestamp last_message_time_ = BootTimestamp::min_time();
Austin Schuh6a7358f2021-11-18 22:40:40 -0800903 BootTimestamp last_popped_message_time_ = BootTimestamp::min_time();
Austin Schuhd2f96102020-12-01 20:27:29 -0800904 // Time this node is queued up until. Used for caching.
Austin Schuh2dc8c7d2021-07-01 17:41:28 -0700905 BootTimestamp queued_until_ = BootTimestamp::min_time();
Austin Schuh79b30942021-01-24 22:32:21 -0800906
907 std::function<void(TimestampedMessage *)> timestamp_callback_;
Eric Schmiedebergb38477e2022-12-02 16:08:04 -0700908 std::function<bool(TimestampedMessage &)> replay_channels_callback_;
Austin Schuh8f52ed52020-11-30 23:12:39 -0800909};
910
Austin Schuhee711052020-08-24 16:06:09 -0700911// Returns the node name with a trailing space, or an empty string if we are on
912// a single node.
913std::string MaybeNodeName(const Node *);
914
Brian Silvermanf51499a2020-09-21 12:49:08 -0700915} // namespace aos::logger
Austin Schuha36c8902019-12-30 18:07:15 -0800916
917#endif // AOS_EVENTS_LOGGING_LOGFILE_UTILS_H_