blob: 5374acfbc8f644970259dcfc442c53fdcda86564 [file] [log] [blame]
Austin Schuhcb5601b2020-09-10 15:29:59 -07001#ifndef AOS_EVENTS_LOGGING_LOG_NAMER_H_
2#define AOS_EVENTS_LOGGING_LOG_NAMER_H_
3
4#include <functional>
5#include <map>
6#include <memory>
7#include <string_view>
8#include <vector>
9
Austin Schuh58646e22021-08-23 23:51:46 -070010#include "absl/container/btree_map.h"
Philipp Schrader790cb542023-07-05 21:06:52 -070011#include "flatbuffers/flatbuffers.h"
Maxwell Gumley8ad77782023-07-11 13:27:03 -060012#include "glog/logging.h"
Philipp Schrader790cb542023-07-05 21:06:52 -070013
Austin Schuhcb5601b2020-09-10 15:29:59 -070014#include "aos/events/logging/logfile_utils.h"
15#include "aos/events/logging/logger_generated.h"
Austin Schuh4385b142021-03-14 21:31:13 -070016#include "aos/uuid.h"
Austin Schuhcb5601b2020-09-10 15:29:59 -070017
18namespace aos {
19namespace logger {
20
Austin Schuh572924a2021-07-30 22:32:12 -070021class LogNamer;
22
Austin Schuhb8bca732021-07-30 22:32:00 -070023// TODO(austin): Rename this back to DataWriter once all other callers are of
24// the old DataWriter.
Austin Schuh572924a2021-07-30 22:32:12 -070025//
26// Class to manage writing data to log files. This lets us track which boot the
27// written header has in it, and if the header has been written or not.
Austin Schuh58646e22021-08-23 23:51:46 -070028//
29// The design of this class is that instead of being notified when any of the
30// header data changes, it polls and owns that decision. This makes it much
31// harder to write corrupted data. If that becomes a performance problem, we
32// can DCHECK and take it out of production binaries.
Austin Schuhb8bca732021-07-30 22:32:00 -070033class NewDataWriter {
34 public:
35 // Constructs a NewDataWriter.
Austin Schuh572924a2021-07-30 22:32:12 -070036 // log_namer is the log namer which holds the config and any other data we
37 // need for our header.
38 // node is the node whom's prespective we are logging from.
Austin Schuhb8bca732021-07-30 22:32:00 -070039 // reopen is called whenever a file needs to be reopened.
40 // close is called to close that file and extract any statistics.
Austin Schuhf5f99f32022-02-07 20:05:37 -080041 NewDataWriter(LogNamer *log_namer, const Node *node, const Node *logger_node,
Austin Schuh572924a2021-07-30 22:32:12 -070042 std::function<void(NewDataWriter *)> reopen,
Austin Schuh48d10d62022-10-16 22:19:23 -070043 std::function<void(NewDataWriter *)> close,
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -070044 size_t max_message_size,
45 std::initializer_list<StoredDataType> types);
Austin Schuh48d10d62022-10-16 22:19:23 -070046
47 void UpdateMaxMessageSize(size_t new_size) {
48 if (new_size > max_message_size_) {
Alexei Strotsbc082d82023-05-03 08:43:42 -070049 CHECK(!header_written_) << ": Tried to update to " << new_size << ", was "
50 << max_message_size_ << " for " << name();
Austin Schuh48d10d62022-10-16 22:19:23 -070051 max_message_size_ = new_size;
52 }
53 }
54 size_t max_message_size() const { return max_message_size_; }
Austin Schuhb8bca732021-07-30 22:32:00 -070055
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -070056 std::chrono::nanoseconds max_out_of_order_duration() const {
57 return max_out_of_order_duration_;
58 }
59
Austin Schuhb8bca732021-07-30 22:32:00 -070060 NewDataWriter(NewDataWriter &&other) = default;
61 aos::logger::NewDataWriter &operator=(NewDataWriter &&other) = default;
62 NewDataWriter(const NewDataWriter &) = delete;
63 void operator=(const NewDataWriter &) = delete;
64
Austin Schuh572924a2021-07-30 22:32:12 -070065 ~NewDataWriter();
Austin Schuhb8bca732021-07-30 22:32:00 -070066
Austin Schuh572924a2021-07-30 22:32:12 -070067 // Rotates the log file, delaying writing the new header until data arrives.
68 void Rotate();
Austin Schuhb8bca732021-07-30 22:32:00 -070069
Austin Schuhf5f99f32022-02-07 20:05:37 -080070 // Updates all the metadata in the log file about the remote node which this
71 // message is from.
Austin Schuh72211ae2021-08-05 14:02:30 -070072 void UpdateRemote(size_t remote_node_index, const UUID &remote_node_boot_uuid,
73 monotonic_clock::time_point monotonic_remote_time,
74 monotonic_clock::time_point monotonic_event_time,
Austin Schuhf5f99f32022-02-07 20:05:37 -080075 bool reliable,
76 monotonic_clock::time_point monotonic_timestamp_time =
77 monotonic_clock::min_time);
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -070078
Austin Schuh48d10d62022-10-16 22:19:23 -070079 // Coppies a message with the provided boot UUID.
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -070080 void CopyDataMessage(DataEncoder::Copier *copier,
81 const UUID &source_node_boot_uuid,
82 aos::monotonic_clock::time_point now,
83 aos::monotonic_clock::time_point message_time);
84 void CopyTimestampMessage(DataEncoder::Copier *copier,
85 const UUID &source_node_boot_uuid,
86 aos::monotonic_clock::time_point now,
87 aos::monotonic_clock::time_point message_time);
88 void CopyRemoteTimestampMessage(
89 DataEncoder::Copier *copier, const UUID &source_node_boot_uuid,
90 aos::monotonic_clock::time_point now,
91 aos::monotonic_clock::time_point message_time);
Austin Schuhb8bca732021-07-30 22:32:00 -070092
Austin Schuh5e14d842022-01-21 12:02:15 -080093 // Updates the current boot for the source node. This is useful when you want
94 // to queue a message that may trigger a reboot rotation, but then need to
95 // update the remote timestamps.
96 void UpdateBoot(const UUID &source_node_boot_uuid);
97
Alexei Strotsbc082d82023-05-03 08:43:42 -070098 // Returns the name of the writer. It may be a filename, but assume it is not.
99 std::string_view name() const { return writer ? writer->name() : "(closed)"; }
Austin Schuhb8bca732021-07-30 22:32:00 -0700100
Austin Schuh572924a2021-07-30 22:32:12 -0700101 void Close();
Austin Schuhb8bca732021-07-30 22:32:00 -0700102
103 std::unique_ptr<DetachedBufferWriter> writer = nullptr;
Austin Schuh572924a2021-07-30 22:32:12 -0700104
105 size_t node_index() const { return node_index_; }
106 const UUID &parts_uuid() const { return parts_uuid_; }
107 size_t parts_index() const { return parts_index_; }
108 const Node *node() const { return node_; }
Austin Schuhb8bca732021-07-30 22:32:00 -0700109
Austin Schuh72211ae2021-08-05 14:02:30 -0700110 // Datastructure used to capture all the information about a remote node.
111 struct State {
112 // Boot UUID of the node.
113 UUID boot_uuid = UUID::Zero();
114 // Timestamp on the remote monotonic clock of the oldest message sent to
115 // node_index_.
116 monotonic_clock::time_point oldest_remote_monotonic_timestamp =
117 monotonic_clock::max_time;
118 // Timestamp on the local monotonic clock of the message in
119 // oldest_remote_monotonic_timestamp.
120 monotonic_clock::time_point oldest_local_monotonic_timestamp =
121 monotonic_clock::max_time;
122 // Timestamp on the remote monotonic clock of the oldest message sent to
123 // node_index_, excluding messages forwarded with time_to_live() == 0.
124 monotonic_clock::time_point oldest_remote_unreliable_monotonic_timestamp =
125 monotonic_clock::max_time;
126 // Timestamp on the local monotonic clock of the message in
127 // oldest_local_unreliable_monotonic_timestamp.
128 monotonic_clock::time_point oldest_local_unreliable_monotonic_timestamp =
129 monotonic_clock::max_time;
Austin Schuhbfe6c572022-01-27 20:48:20 -0800130
131 // Timestamp on the remote monotonic clock of the oldest message sent to
132 // node_index_, only including messages forwarded with time_to_live() == 0.
133 monotonic_clock::time_point oldest_remote_reliable_monotonic_timestamp =
134 monotonic_clock::max_time;
135 // Timestamp on the local monotonic clock of the message in
136 // oldest_local_reliable_monotonic_timestamp.
137 monotonic_clock::time_point oldest_local_reliable_monotonic_timestamp =
138 monotonic_clock::max_time;
Austin Schuhf5f99f32022-02-07 20:05:37 -0800139
140 // Timestamp on the remote monotonic clock of the oldest message timestamp
141 // sent back to logger_node_index_. The remote here will be the node this
142 // part is from the perspective of, ie node_index_.
143 monotonic_clock::time_point
144 oldest_logger_remote_unreliable_monotonic_timestamp =
145 monotonic_clock::max_time;
146 // The time on the monotonic clock of the logger when this timestamp made it
147 // back to the logger (logger_node_index_).
148 monotonic_clock::time_point
149 oldest_logger_local_unreliable_monotonic_timestamp =
150 monotonic_clock::max_time;
Austin Schuh72211ae2021-08-05 14:02:30 -0700151 };
152
Austin Schuhb8bca732021-07-30 22:32:00 -0700153 private:
Austin Schuhe46492f2021-07-31 19:49:41 -0700154 // Signals that a node has rebooted.
Austin Schuh5e14d842022-01-21 12:02:15 -0800155 void Reboot(const UUID &source_node_boot_uuid);
Austin Schuhe46492f2021-07-31 19:49:41 -0700156
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -0700157 void CopyMessage(DataEncoder::Copier *copier,
158 const UUID &source_node_boot_uuid,
159 aos::monotonic_clock::time_point now,
160 aos::monotonic_clock::time_point message_time);
161
Austin Schuh572924a2021-07-30 22:32:12 -0700162 void QueueHeader(
163 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> &&header);
164
Austin Schuhe46492f2021-07-31 19:49:41 -0700165 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader();
166
Austin Schuh58646e22021-08-23 23:51:46 -0700167 monotonic_clock::time_point monotonic_start_time_ = monotonic_clock::min_time;
168
Austin Schuh577610e2021-12-08 12:07:19 -0800169 const Node *node_ = nullptr;
170 size_t node_index_ = 0;
Austin Schuhf5f99f32022-02-07 20:05:37 -0800171 size_t logger_node_index_ = 0;
Austin Schuh572924a2021-07-30 22:32:12 -0700172 LogNamer *log_namer_;
173 UUID parts_uuid_ = UUID::Random();
174 size_t parts_index_ = 0;
175
Austin Schuhb8bca732021-07-30 22:32:00 -0700176 std::function<void(NewDataWriter *)> reopen_;
177 std::function<void(NewDataWriter *)> close_;
Austin Schuh572924a2021-07-30 22:32:12 -0700178 bool header_written_ = false;
Austin Schuhe46492f2021-07-31 19:49:41 -0700179
Austin Schuh72211ae2021-08-05 14:02:30 -0700180 std::vector<State> state_;
Austin Schuh48d10d62022-10-16 22:19:23 -0700181
182 size_t max_message_size_;
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -0700183
184 // Each data writer logs the channels for that node, i.e.
185 // each data writer writes one file. We may encounter messages which
186 // violate the max out of order duration specified in the header of that file.
187 // Rotate the data writer and start a new part for that particular file.
188 // This shouldn't affect the headers of other data writers, so make this
189 // a property of individual data writer instead of the overall log.
190 std::chrono::nanoseconds max_out_of_order_duration_;
191
192 // Monotonic time point of the latest message we've logged so far, i.e
193 // Message X - time Z
194 // Message Y - time Z + 1
195 // newest_message_time_ = Z + 1 (even if X was logged after Y)
196 //
197 // Since the messages can be logged out of order, this helps determine if
198 // max out of order duration was violated.
199 monotonic_clock::time_point newest_message_time_ = monotonic_clock::min_time;
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -0700200
201 // An array with a bool for each value of StoredDataType representing if that
202 // data type is allowed to be logged by this object.
203 std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1>
204 allowed_data_types_;
Austin Schuhb8bca732021-07-30 22:32:00 -0700205};
206
Austin Schuhcb5601b2020-09-10 15:29:59 -0700207// Interface describing how to name, track, and add headers to log file parts.
208class LogNamer {
209 public:
210 // Constructs a LogNamer with the primary node (ie the one the logger runs on)
211 // being node.
Austin Schuh5b728b72021-06-16 14:57:15 -0700212 LogNamer(const aos::Configuration *configuration, EventLoop *event_loop,
213 const aos::Node *node)
Austin Schuha499cea2021-07-31 19:49:53 -0700214 : event_loop_(event_loop),
Austin Schuh5b728b72021-06-16 14:57:15 -0700215 configuration_(configuration),
216 node_(node),
Austin Schuha499cea2021-07-31 19:49:53 -0700217 logger_node_index_(configuration::GetNodeIndex(configuration_, node_)) {
Austin Schuh73340842021-07-30 22:32:06 -0700218 nodes_.emplace_back(node_);
Austin Schuh73340842021-07-30 22:32:06 -0700219 }
Alexei Strots01395492023-03-20 13:59:56 -0700220 virtual ~LogNamer() = default;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700221
Brian Silverman87ac0402020-09-17 14:47:01 -0700222 // Returns a writer for writing data from messages on this channel (on the
223 // primary node).
224 //
225 // The returned pointer will stay valid across rotations, but the object it
226 // points to will be assigned to.
Austin Schuhb8bca732021-07-30 22:32:00 -0700227 virtual NewDataWriter *MakeWriter(const Channel *channel) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700228
Brian Silverman87ac0402020-09-17 14:47:01 -0700229 // Returns a writer for writing timestamps from messages on this channel (on
230 // the primary node).
231 //
232 // The returned pointer will stay valid across rotations, but the object it
233 // points to will be assigned to.
Austin Schuhb8bca732021-07-30 22:32:00 -0700234 virtual NewDataWriter *MakeTimestampWriter(const Channel *channel) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700235
236 // Returns a writer for writing timestamps delivered over the special
237 // /aos/remote_timestamps/* channels. node is the node that the timestamps
Brian Silverman87ac0402020-09-17 14:47:01 -0700238 // are forwarded back from (to the primary node).
239 //
240 // The returned pointer will stay valid across rotations, but the object it
241 // points to will be assigned to.
Austin Schuh73340842021-07-30 22:32:06 -0700242 virtual NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel,
243 const Node *node) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700244
Austin Schuh73340842021-07-30 22:32:06 -0700245 // Rotates all log files for the provided node.
246 virtual void Rotate(const Node *node) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700247
248 // Returns all the nodes that data is being written for.
249 const std::vector<const Node *> &nodes() const { return nodes_; }
250
Austin Schuh08dba8f2023-05-01 08:29:30 -0700251 // Closes all existing log data writers. No more data may be written after
252 // this.
253 virtual WriteCode Close() = 0;
254
Austin Schuhcb5601b2020-09-10 15:29:59 -0700255 // Returns the node the logger is running on.
256 const Node *node() const { return node_; }
Austin Schuhe46492f2021-07-31 19:49:41 -0700257 const UUID &logger_node_boot_uuid() const { return logger_node_boot_uuid_; }
258 size_t logger_node_index() const { return logger_node_index_; }
Austin Schuhcb5601b2020-09-10 15:29:59 -0700259
Austin Schuh8c399962020-12-25 21:51:45 -0800260 // Writes out the nested Configuration object to the config file location.
261 virtual void WriteConfiguration(
262 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header,
263 std::string_view config_sha256) = 0;
264
Austin Schuh73340842021-07-30 22:32:06 -0700265 void SetHeaderTemplate(
266 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header) {
267 header_ = std::move(header);
Austin Schuhe46492f2021-07-31 19:49:41 -0700268 logger_node_boot_uuid_ =
269 UUID::FromString(header_.message().logger_node_boot_uuid());
Austin Schuh73340842021-07-30 22:32:06 -0700270 }
Austin Schuhcb5601b2020-09-10 15:29:59 -0700271
Austin Schuh60e77942022-05-16 17:48:24 -0700272 void ClearStartTimes() { node_states_.clear(); }
Austin Schuh58646e22021-08-23 23:51:46 -0700273
274 void SetStartTimes(size_t node_index, const UUID &boot_uuid,
Austin Schuh73340842021-07-30 22:32:06 -0700275 monotonic_clock::time_point monotonic_start_time,
276 realtime_clock::time_point realtime_start_time,
277 monotonic_clock::time_point logger_monotonic_start_time,
278 realtime_clock::time_point logger_realtime_start_time) {
Austin Schuh58646e22021-08-23 23:51:46 -0700279 VLOG(1) << "Setting node " << node_index << " to start time "
280 << monotonic_start_time << " rt " << realtime_start_time << " UUID "
281 << boot_uuid;
282 NodeState *node_state = GetNodeState(node_index, boot_uuid);
283 node_state->monotonic_start_time = monotonic_start_time;
284 node_state->realtime_start_time = realtime_start_time;
285 node_state->logger_monotonic_start_time = logger_monotonic_start_time;
286 node_state->logger_realtime_start_time = logger_realtime_start_time;
Austin Schuh73340842021-07-30 22:32:06 -0700287 }
288
Austin Schuh58646e22021-08-23 23:51:46 -0700289 monotonic_clock::time_point monotonic_start_time(size_t node_index,
290 const UUID &boot_uuid) {
291 DCHECK_NE(boot_uuid, UUID::Zero());
292
293 NodeState *node_state = GetNodeState(node_index, boot_uuid);
294 return node_state->monotonic_start_time;
Austin Schuh73340842021-07-30 22:32:06 -0700295 }
296
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -0700297 // This returns the initial out of order duration set in the header template
298 // by the logger based on polling period. It may be different than the actual
299 // duration used by the data writer.
300 std::chrono::nanoseconds base_max_out_of_order_duration() const {
301 return std::chrono::nanoseconds(
302 header_.message().max_out_of_order_duration());
303 }
304
Austin Schuh73340842021-07-30 22:32:06 -0700305 protected:
Austin Schuh73340842021-07-30 22:32:06 -0700306 // Structure with state per node about times and such.
Austin Schuh73340842021-07-30 22:32:06 -0700307 struct NodeState {
308 // Time when this node started logging.
309 monotonic_clock::time_point monotonic_start_time =
310 monotonic_clock::min_time;
311 realtime_clock::time_point realtime_start_time = realtime_clock::min_time;
312
313 // Corresponding time on the logger node when it started logging.
314 monotonic_clock::time_point logger_monotonic_start_time =
315 monotonic_clock::min_time;
316 realtime_clock::time_point logger_realtime_start_time =
317 realtime_clock::min_time;
Austin Schuh73340842021-07-30 22:32:06 -0700318 };
Austin Schuh58646e22021-08-23 23:51:46 -0700319
320 // Creates a new header by copying fields out of the template and combining
321 // them with the arguments provided.
322 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader(
323 size_t node_index, const std::vector<NewDataWriter::State> &state,
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -0700324 const UUID &parts_uuid, int parts_index,
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -0700325 std::chrono::nanoseconds max_out_of_order_duration,
326 const std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1>
327 &allowed_data_types);
Austin Schuh58646e22021-08-23 23:51:46 -0700328
329 EventLoop *event_loop_;
330 const Configuration *const configuration_;
331 const Node *const node_;
332 const size_t logger_node_index_;
333 UUID logger_node_boot_uuid_;
334 std::vector<const Node *> nodes_;
335
336 friend NewDataWriter;
337
338 // Returns the start/stop time state structure for a node and boot. We can
339 // have data from multiple boots, and it makes sense to reuse the start/stop
340 // times if we get data from the same boot again.
341 NodeState *GetNodeState(size_t node_index, const UUID &boot_uuid);
342
343 absl::btree_map<std::pair<size_t, UUID>, NodeState> node_states_;
Austin Schuh73340842021-07-30 22:32:06 -0700344
345 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header_ =
346 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader>::Empty();
Austin Schuhcb5601b2020-09-10 15:29:59 -0700347};
348
Alexei Strots01395492023-03-20 13:59:56 -0700349// Log namer which uses a config to name a bunch of files.
Austin Schuhcb5601b2020-09-10 15:29:59 -0700350class MultiNodeLogNamer : public LogNamer {
351 public:
Alexei Strotscaf17d32023-04-03 22:31:11 -0700352 MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend,
Alexei Strots01395492023-03-20 13:59:56 -0700353 EventLoop *event_loop);
Alexei Strotscaf17d32023-04-03 22:31:11 -0700354 MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend,
Austin Schuh5b728b72021-06-16 14:57:15 -0700355 const Configuration *configuration, EventLoop *event_loop,
356 const Node *node);
Brian Silvermancb805822020-10-06 17:43:35 -0700357 ~MultiNodeLogNamer() override;
358
Austin Schuh48d10d62022-10-16 22:19:23 -0700359 // Sets the function for creating encoders. The argument is the max message
360 // size (including headers) that will be written into this encoder.
Brian Silvermancb805822020-10-06 17:43:35 -0700361 //
362 // Defaults to just creating DummyEncoders.
363 void set_encoder_factory(
Austin Schuh48d10d62022-10-16 22:19:23 -0700364 std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory) {
Brian Silvermancb805822020-10-06 17:43:35 -0700365 encoder_factory_ = std::move(encoder_factory);
366 }
367
368 // Sets an additional file extension.
369 //
370 // Defaults to nothing.
371 void set_extension(std::string_view extension) { extension_ = extension; }
Brian Silverman1f345222020-09-24 21:14:48 -0700372
Brian Silvermana621f522020-09-30 16:52:43 -0700373 // A list of all the filenames we've written.
374 //
375 // This only includes the part after base_name().
376 const std::vector<std::string> &all_filenames() const {
377 return all_filenames_;
378 }
379
Austin Schuh73340842021-07-30 22:32:06 -0700380 void Rotate(const Node *node) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700381
Austin Schuh8c399962020-12-25 21:51:45 -0800382 void WriteConfiguration(
383 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header,
384 std::string_view config_sha256) override;
385
Austin Schuhb8bca732021-07-30 22:32:00 -0700386 NewDataWriter *MakeWriter(const Channel *channel) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700387
Austin Schuhb8bca732021-07-30 22:32:00 -0700388 NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel,
Austin Schuh73340842021-07-30 22:32:06 -0700389 const Node *node) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700390
Austin Schuhb8bca732021-07-30 22:32:00 -0700391 NewDataWriter *MakeTimestampWriter(const Channel *channel) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700392
Brian Silverman0465fcf2020-09-24 00:29:18 -0700393 // Indicates that at least one file ran out of space. Once this happens, we
394 // stop trying to open new files, to avoid writing any files with holes from
395 // previous parts.
396 //
397 // Besides this function, this object will silently stop logging data when
398 // this occurs. If you want to ensure log files are complete, you must call
399 // this method.
Brian Silvermana9f2ec92020-10-06 18:00:53 -0700400 bool ran_out_of_space() const {
401 return accumulate_data_writers<bool>(
Austin Schuhb8bca732021-07-30 22:32:00 -0700402 ran_out_of_space_, [](bool x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600403 CHECK_NOTNULL(data_writer.writer);
Brian Silvermana9f2ec92020-10-06 18:00:53 -0700404 return x ||
405 (data_writer.writer && data_writer.writer->ran_out_of_space());
406 });
407 }
Brian Silverman0465fcf2020-09-24 00:29:18 -0700408
Brian Silverman1f345222020-09-24 21:14:48 -0700409 // Returns the maximum total_bytes() value for all existing
410 // DetachedBufferWriters.
411 //
412 // Returns 0 if no files are open.
413 size_t maximum_total_bytes() const {
Brian Silvermancb805822020-10-06 17:43:35 -0700414 return accumulate_data_writers<size_t>(
Austin Schuhb8bca732021-07-30 22:32:00 -0700415 0, [](size_t x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600416 CHECK_NOTNULL(data_writer.writer);
Brian Silvermancb805822020-10-06 17:43:35 -0700417 return std::max(x, data_writer.writer->total_bytes());
418 });
Brian Silverman1f345222020-09-24 21:14:48 -0700419 }
420
Brian Silverman0465fcf2020-09-24 00:29:18 -0700421 // Closes all existing log files. No more data may be written after this.
422 //
423 // This may set ran_out_of_space().
Austin Schuh08dba8f2023-05-01 08:29:30 -0700424 WriteCode Close() override;
Brian Silverman0465fcf2020-09-24 00:29:18 -0700425
Brian Silvermancb805822020-10-06 17:43:35 -0700426 // Accessors for various statistics. See the identically-named methods in
427 // DetachedBufferWriter for documentation. These are aggregated across all
428 // past and present DetachedBufferWriters.
429 std::chrono::nanoseconds max_write_time() const {
430 return accumulate_data_writers(
431 max_write_time_,
Austin Schuhb8bca732021-07-30 22:32:00 -0700432 [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600433 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700434 return std::max(
435 x, data_writer.writer->WriteStatistics()->max_write_time());
Brian Silvermancb805822020-10-06 17:43:35 -0700436 });
437 }
438 int max_write_time_bytes() const {
439 return std::get<0>(accumulate_data_writers(
440 std::make_tuple(max_write_time_bytes_, max_write_time_),
441 [](std::tuple<int, std::chrono::nanoseconds> x,
Austin Schuhb8bca732021-07-30 22:32:00 -0700442 const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600443 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700444 if (data_writer.writer->WriteStatistics()->max_write_time() >
445 std::get<1>(x)) {
446 return std::make_tuple(
447 data_writer.writer->WriteStatistics()->max_write_time_bytes(),
448 data_writer.writer->WriteStatistics()->max_write_time());
Brian Silvermancb805822020-10-06 17:43:35 -0700449 }
450 return x;
451 }));
452 }
453 int max_write_time_messages() const {
454 return std::get<0>(accumulate_data_writers(
455 std::make_tuple(max_write_time_messages_, max_write_time_),
456 [](std::tuple<int, std::chrono::nanoseconds> x,
Austin Schuhb8bca732021-07-30 22:32:00 -0700457 const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600458 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700459 if (data_writer.writer->WriteStatistics()->max_write_time() >
460 std::get<1>(x)) {
Brian Silvermancb805822020-10-06 17:43:35 -0700461 return std::make_tuple(
Alexei Strots01395492023-03-20 13:59:56 -0700462 data_writer.writer->WriteStatistics()
463 ->max_write_time_messages(),
464 data_writer.writer->WriteStatistics()->max_write_time());
Brian Silvermancb805822020-10-06 17:43:35 -0700465 }
466 return x;
467 }));
468 }
469 std::chrono::nanoseconds total_write_time() const {
470 return accumulate_data_writers(
471 total_write_time_,
Austin Schuhb8bca732021-07-30 22:32:00 -0700472 [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600473 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700474 return x + data_writer.writer->WriteStatistics()->total_write_time();
Brian Silvermancb805822020-10-06 17:43:35 -0700475 });
476 }
477 int total_write_count() const {
478 return accumulate_data_writers(
Austin Schuhb8bca732021-07-30 22:32:00 -0700479 total_write_count_, [](int x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600480 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700481 return x + data_writer.writer->WriteStatistics()->total_write_count();
Brian Silvermancb805822020-10-06 17:43:35 -0700482 });
483 }
484 int total_write_messages() const {
485 return accumulate_data_writers(
Austin Schuhb8bca732021-07-30 22:32:00 -0700486 total_write_messages_, [](int x, const NewDataWriter &data_writer) {
Alexei Strots01395492023-03-20 13:59:56 -0700487 return x +
488 data_writer.writer->WriteStatistics()->total_write_messages();
Brian Silvermancb805822020-10-06 17:43:35 -0700489 });
490 }
491 int total_write_bytes() const {
492 return accumulate_data_writers(
Austin Schuhb8bca732021-07-30 22:32:00 -0700493 total_write_bytes_, [](int x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600494 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700495 return x + data_writer.writer->WriteStatistics()->total_write_bytes();
Brian Silvermancb805822020-10-06 17:43:35 -0700496 });
497 }
498
499 void ResetStatistics();
500
Alexei Strotscaf17d32023-04-03 22:31:11 -0700501 protected:
502 // TODO (Alexei): consider to move ownership of log_namer to concrete sub
503 // class and make log_backend_ raw pointer.
504 LogBackend *log_backend() { return log_backend_.get(); }
505 const LogBackend *log_backend() const { return log_backend_.get(); }
506
Austin Schuh6ecfe902023-08-04 22:44:37 -0700507 // Returns the data writer or timestamp writer if we find one for the provided
508 // node.
509 NewDataWriter *FindNodeDataWriter(const Node *node, size_t max_message_size);
510 NewDataWriter *FindNodeTimestampWriter(const Node *node,
511 size_t max_message_size);
512
513 // Saves the data writer or timestamp writer for the provided node.
514 NewDataWriter *AddNodeDataWriter(const Node *node, NewDataWriter &&writer);
515 NewDataWriter *AddNodeTimestampWriter(const Node *node,
516 NewDataWriter &&writer);
517
518 void CloseWriter(std::unique_ptr<DetachedBufferWriter> *writer_pointer);
519
520 void CreateBufferWriter(std::string_view path, size_t max_message_size,
521 std::unique_ptr<DetachedBufferWriter> *destination);
522
523 std::string extension_;
524
Austin Schuhcb5601b2020-09-10 15:29:59 -0700525 private:
Austin Schuhcb5601b2020-09-10 15:29:59 -0700526 // Opens up a writer for timestamps forwarded back.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700527 void OpenForwardedTimestampWriter(const Node *source_node,
Austin Schuhb8bca732021-07-30 22:32:00 -0700528 NewDataWriter *data_writer);
Austin Schuhcb5601b2020-09-10 15:29:59 -0700529
530 // Opens up a writer for remote data.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700531 void OpenDataWriter(const Node *source_node, NewDataWriter *data_writer);
532 void OpenTimestampWriter(NewDataWriter *data_writer);
Austin Schuhcb5601b2020-09-10 15:29:59 -0700533
Austin Schuh6ecfe902023-08-04 22:44:37 -0700534 // Tracks the node in nodes_.
535 void NoticeNode(const Node *source_node);
Austin Schuhcb5601b2020-09-10 15:29:59 -0700536
Brian Silvermancb805822020-10-06 17:43:35 -0700537 // A version of std::accumulate which operates over all of our DataWriters.
538 template <typename T, typename BinaryOperation>
539 T accumulate_data_writers(T t, BinaryOperation op) const {
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700540 for (const std::pair<const Node *const, NewDataWriter> &data_writer :
541 node_data_writers_) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600542 if (data_writer.second.writer != nullptr) {
543 t = op(std::move(t), data_writer.second);
544 }
Brian Silvermancb805822020-10-06 17:43:35 -0700545 }
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700546 for (const std::pair<const Node *const, NewDataWriter> &data_writer :
547 node_timestamp_writers_) {
548 if (data_writer.second.writer != nullptr) {
549 t = op(std::move(t), data_writer.second);
550 }
Brian Silvermancb805822020-10-06 17:43:35 -0700551 }
552 return t;
553 }
554
Alexei Strotscaf17d32023-04-03 22:31:11 -0700555 std::unique_ptr<LogBackend> log_backend_;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700556
Brian Silverman0465fcf2020-09-24 00:29:18 -0700557 bool ran_out_of_space_ = false;
Brian Silvermana621f522020-09-30 16:52:43 -0700558 std::vector<std::string> all_filenames_;
Brian Silverman0465fcf2020-09-24 00:29:18 -0700559
Austin Schuh8bdfc492023-02-11 12:53:13 -0800560 std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory_;
Brian Silvermancb805822020-10-06 17:43:35 -0700561
562 // Storage for statistics from previously-rotated DetachedBufferWriters.
563 std::chrono::nanoseconds max_write_time_ = std::chrono::nanoseconds::zero();
564 int max_write_time_bytes_ = -1;
565 int max_write_time_messages_ = -1;
566 std::chrono::nanoseconds total_write_time_ = std::chrono::nanoseconds::zero();
567 int total_write_count_ = 0;
568 int total_write_messages_ = 0;
569 int total_write_bytes_ = 0;
570
Mithun Bharadwaj0c629932023-08-02 16:10:40 -0700571 // Data writer per remote node.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700572 std::map<const Node *, NewDataWriter> node_data_writers_;
Mithun Bharadwaj99aec9e2023-08-02 16:10:40 -0700573 // Remote timestamp writers per node.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700574 std::map<const Node *, NewDataWriter> node_timestamp_writers_;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700575};
576
Alexei Strots01395492023-03-20 13:59:56 -0700577// This is specialized log namer that deals with directory centric log events.
578class MultiNodeFilesLogNamer : public MultiNodeLogNamer {
579 public:
580 MultiNodeFilesLogNamer(std::string_view base_name, EventLoop *event_loop)
colleen61276dc2023-06-01 09:23:29 -0700581 : MultiNodeLogNamer(
582 std::make_unique<RenamableFileBackend>(base_name, false),
583 event_loop) {}
Alexei Strots01395492023-03-20 13:59:56 -0700584
585 MultiNodeFilesLogNamer(std::string_view base_name,
586 const Configuration *configuration,
587 EventLoop *event_loop, const Node *node)
colleen61276dc2023-06-01 09:23:29 -0700588 : MultiNodeLogNamer(
589 std::make_unique<RenamableFileBackend>(base_name, false),
590 configuration, event_loop, node) {}
591
592 MultiNodeFilesLogNamer(EventLoop *event_loop,
593 std::unique_ptr<RenamableFileBackend> backend)
594 : MultiNodeLogNamer(std::move(backend), event_loop) {}
595
Alexei Strots01395492023-03-20 13:59:56 -0700596 ~MultiNodeFilesLogNamer() override = default;
Alexei Strotscaf17d32023-04-03 22:31:11 -0700597
598 std::string_view base_name() const {
599 return renamable_file_backend()->base_name();
600 }
601
602 // Rotate should be called at least once in between calls to set_base_name.
603 // Otherwise, temporary files will not be recoverable.
604 // Rotate is called by Logger::RenameLogBase, which is currently the only user
605 // of this method.
606 // Only renaming the folder is supported, not the file base name.
607 void set_base_name(std::string_view base_name) {
608 renamable_file_backend()->RenameLogBase(base_name);
609 }
610
611 // When enabled, this will write files under names beginning
612 // with the .tmp suffix, and then rename them to the desired name after
613 // they are fully written.
614 //
615 // This is useful to enable incremental copying of the log files.
616 //
617 // Defaults to writing directly to the final filename.
618 void EnableTempFiles() { renamable_file_backend()->EnableTempFiles(); }
619
620 private:
621 RenamableFileBackend *renamable_file_backend() {
622 return reinterpret_cast<RenamableFileBackend *>(log_backend());
623 }
624 const RenamableFileBackend *renamable_file_backend() const {
625 return reinterpret_cast<const RenamableFileBackend *>(log_backend());
626 }
Alexei Strots01395492023-03-20 13:59:56 -0700627};
628
Austin Schuh6ecfe902023-08-04 22:44:37 -0700629// Class which dumps all data from each node into a single file per node. This
630// is mostly interesting for testing.
631class MinimalFileMultiNodeLogNamer : public MultiNodeFilesLogNamer {
632 public:
633 MinimalFileMultiNodeLogNamer(std::string_view base_name,
634 EventLoop *event_loop)
635 : MultiNodeFilesLogNamer(base_name, event_loop) {}
636 MinimalFileMultiNodeLogNamer(std::string_view base_name,
637 const Configuration *configuration,
638 EventLoop *event_loop, const Node *node)
639 : MultiNodeFilesLogNamer(base_name, configuration, event_loop, node) {}
640
641 NewDataWriter *MakeWriter(const Channel *channel) override;
642
643 NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel,
644 const Node *node) override;
645
646 NewDataWriter *MakeTimestampWriter(const Channel *channel) override;
647
648 private:
649 // Names the data writer.
650 void OpenNodeWriter(const Node *source_node, NewDataWriter *data_writer);
651};
652
Austin Schuhcb5601b2020-09-10 15:29:59 -0700653} // namespace logger
654} // namespace aos
655
656#endif // AOS_EVENTS_LOGGING_LOG_NAMER_H_