blob: e0660eb2d9048d157a286b423e0cb79ee2ba44ab [file] [log] [blame]
Austin Schuhcb5601b2020-09-10 15:29:59 -07001#ifndef AOS_EVENTS_LOGGING_LOG_NAMER_H_
2#define AOS_EVENTS_LOGGING_LOG_NAMER_H_
3
4#include <functional>
5#include <map>
6#include <memory>
7#include <string_view>
8#include <vector>
9
Austin Schuh58646e22021-08-23 23:51:46 -070010#include "absl/container/btree_map.h"
Philipp Schrader790cb542023-07-05 21:06:52 -070011#include "flatbuffers/flatbuffers.h"
Maxwell Gumley8ad77782023-07-11 13:27:03 -060012#include "glog/logging.h"
Philipp Schrader790cb542023-07-05 21:06:52 -070013
Austin Schuhcb5601b2020-09-10 15:29:59 -070014#include "aos/events/logging/logfile_utils.h"
15#include "aos/events/logging/logger_generated.h"
Austin Schuh4385b142021-03-14 21:31:13 -070016#include "aos/uuid.h"
Austin Schuhcb5601b2020-09-10 15:29:59 -070017
Stephan Pleinesd99b1ee2024-02-02 20:56:44 -080018namespace aos::logger {
Austin Schuhcb5601b2020-09-10 15:29:59 -070019
Austin Schuh572924a2021-07-30 22:32:12 -070020class LogNamer;
21
Austin Schuhb8bca732021-07-30 22:32:00 -070022// TODO(austin): Rename this back to DataWriter once all other callers are of
23// the old DataWriter.
Austin Schuh572924a2021-07-30 22:32:12 -070024//
25// Class to manage writing data to log files. This lets us track which boot the
26// written header has in it, and if the header has been written or not.
Austin Schuh58646e22021-08-23 23:51:46 -070027//
28// The design of this class is that instead of being notified when any of the
29// header data changes, it polls and owns that decision. This makes it much
30// harder to write corrupted data. If that becomes a performance problem, we
31// can DCHECK and take it out of production binaries.
Austin Schuhb8bca732021-07-30 22:32:00 -070032class NewDataWriter {
33 public:
34 // Constructs a NewDataWriter.
Austin Schuh572924a2021-07-30 22:32:12 -070035 // log_namer is the log namer which holds the config and any other data we
36 // need for our header.
37 // node is the node whom's prespective we are logging from.
Austin Schuhb8bca732021-07-30 22:32:00 -070038 // reopen is called whenever a file needs to be reopened.
39 // close is called to close that file and extract any statistics.
Austin Schuhf5f99f32022-02-07 20:05:37 -080040 NewDataWriter(LogNamer *log_namer, const Node *node, const Node *logger_node,
Austin Schuh572924a2021-07-30 22:32:12 -070041 std::function<void(NewDataWriter *)> reopen,
Austin Schuh48d10d62022-10-16 22:19:23 -070042 std::function<void(NewDataWriter *)> close,
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -070043 size_t max_message_size,
44 std::initializer_list<StoredDataType> types);
Austin Schuh48d10d62022-10-16 22:19:23 -070045
46 void UpdateMaxMessageSize(size_t new_size) {
47 if (new_size > max_message_size_) {
Alexei Strotsbc082d82023-05-03 08:43:42 -070048 CHECK(!header_written_) << ": Tried to update to " << new_size << ", was "
49 << max_message_size_ << " for " << name();
Austin Schuh48d10d62022-10-16 22:19:23 -070050 max_message_size_ = new_size;
51 }
52 }
53 size_t max_message_size() const { return max_message_size_; }
Austin Schuhb8bca732021-07-30 22:32:00 -070054
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -070055 std::chrono::nanoseconds max_out_of_order_duration() const {
56 return max_out_of_order_duration_;
57 }
58
Austin Schuhb8bca732021-07-30 22:32:00 -070059 NewDataWriter(NewDataWriter &&other) = default;
60 aos::logger::NewDataWriter &operator=(NewDataWriter &&other) = default;
61 NewDataWriter(const NewDataWriter &) = delete;
62 void operator=(const NewDataWriter &) = delete;
63
Austin Schuh572924a2021-07-30 22:32:12 -070064 ~NewDataWriter();
Austin Schuhb8bca732021-07-30 22:32:00 -070065
Austin Schuh572924a2021-07-30 22:32:12 -070066 // Rotates the log file, delaying writing the new header until data arrives.
67 void Rotate();
Austin Schuhb8bca732021-07-30 22:32:00 -070068
Austin Schuhf5f99f32022-02-07 20:05:37 -080069 // Updates all the metadata in the log file about the remote node which this
70 // message is from.
Austin Schuh72211ae2021-08-05 14:02:30 -070071 void UpdateRemote(size_t remote_node_index, const UUID &remote_node_boot_uuid,
72 monotonic_clock::time_point monotonic_remote_time,
73 monotonic_clock::time_point monotonic_event_time,
Austin Schuhf5f99f32022-02-07 20:05:37 -080074 bool reliable,
75 monotonic_clock::time_point monotonic_timestamp_time =
76 monotonic_clock::min_time);
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -070077
Austin Schuh48d10d62022-10-16 22:19:23 -070078 // Coppies a message with the provided boot UUID.
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -070079 void CopyDataMessage(DataEncoder::Copier *copier,
80 const UUID &source_node_boot_uuid,
81 aos::monotonic_clock::time_point now,
82 aos::monotonic_clock::time_point message_time);
83 void CopyTimestampMessage(DataEncoder::Copier *copier,
84 const UUID &source_node_boot_uuid,
85 aos::monotonic_clock::time_point now,
86 aos::monotonic_clock::time_point message_time);
87 void CopyRemoteTimestampMessage(
88 DataEncoder::Copier *copier, const UUID &source_node_boot_uuid,
89 aos::monotonic_clock::time_point now,
90 aos::monotonic_clock::time_point message_time);
Austin Schuhb8bca732021-07-30 22:32:00 -070091
Austin Schuh5e14d842022-01-21 12:02:15 -080092 // Updates the current boot for the source node. This is useful when you want
93 // to queue a message that may trigger a reboot rotation, but then need to
94 // update the remote timestamps.
95 void UpdateBoot(const UUID &source_node_boot_uuid);
96
Alexei Strotsbc082d82023-05-03 08:43:42 -070097 // Returns the name of the writer. It may be a filename, but assume it is not.
98 std::string_view name() const { return writer ? writer->name() : "(closed)"; }
Austin Schuhb8bca732021-07-30 22:32:00 -070099
Austin Schuh572924a2021-07-30 22:32:12 -0700100 void Close();
Austin Schuhb8bca732021-07-30 22:32:00 -0700101
102 std::unique_ptr<DetachedBufferWriter> writer = nullptr;
Austin Schuh572924a2021-07-30 22:32:12 -0700103
104 size_t node_index() const { return node_index_; }
105 const UUID &parts_uuid() const { return parts_uuid_; }
106 size_t parts_index() const { return parts_index_; }
107 const Node *node() const { return node_; }
Austin Schuhb8bca732021-07-30 22:32:00 -0700108
Austin Schuh72211ae2021-08-05 14:02:30 -0700109 // Datastructure used to capture all the information about a remote node.
110 struct State {
111 // Boot UUID of the node.
112 UUID boot_uuid = UUID::Zero();
113 // Timestamp on the remote monotonic clock of the oldest message sent to
114 // node_index_.
115 monotonic_clock::time_point oldest_remote_monotonic_timestamp =
116 monotonic_clock::max_time;
117 // Timestamp on the local monotonic clock of the message in
118 // oldest_remote_monotonic_timestamp.
119 monotonic_clock::time_point oldest_local_monotonic_timestamp =
120 monotonic_clock::max_time;
121 // Timestamp on the remote monotonic clock of the oldest message sent to
122 // node_index_, excluding messages forwarded with time_to_live() == 0.
123 monotonic_clock::time_point oldest_remote_unreliable_monotonic_timestamp =
124 monotonic_clock::max_time;
125 // Timestamp on the local monotonic clock of the message in
126 // oldest_local_unreliable_monotonic_timestamp.
127 monotonic_clock::time_point oldest_local_unreliable_monotonic_timestamp =
128 monotonic_clock::max_time;
Austin Schuhbfe6c572022-01-27 20:48:20 -0800129
130 // Timestamp on the remote monotonic clock of the oldest message sent to
131 // node_index_, only including messages forwarded with time_to_live() == 0.
132 monotonic_clock::time_point oldest_remote_reliable_monotonic_timestamp =
133 monotonic_clock::max_time;
134 // Timestamp on the local monotonic clock of the message in
135 // oldest_local_reliable_monotonic_timestamp.
136 monotonic_clock::time_point oldest_local_reliable_monotonic_timestamp =
137 monotonic_clock::max_time;
Austin Schuhf5f99f32022-02-07 20:05:37 -0800138
139 // Timestamp on the remote monotonic clock of the oldest message timestamp
140 // sent back to logger_node_index_. The remote here will be the node this
141 // part is from the perspective of, ie node_index_.
142 monotonic_clock::time_point
143 oldest_logger_remote_unreliable_monotonic_timestamp =
144 monotonic_clock::max_time;
145 // The time on the monotonic clock of the logger when this timestamp made it
146 // back to the logger (logger_node_index_).
147 monotonic_clock::time_point
148 oldest_logger_local_unreliable_monotonic_timestamp =
149 monotonic_clock::max_time;
Austin Schuh72211ae2021-08-05 14:02:30 -0700150 };
151
Austin Schuhb8bca732021-07-30 22:32:00 -0700152 private:
Austin Schuhe46492f2021-07-31 19:49:41 -0700153 // Signals that a node has rebooted.
Austin Schuh5e14d842022-01-21 12:02:15 -0800154 void Reboot(const UUID &source_node_boot_uuid);
Austin Schuhe46492f2021-07-31 19:49:41 -0700155
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -0700156 void CopyMessage(DataEncoder::Copier *copier,
157 const UUID &source_node_boot_uuid,
158 aos::monotonic_clock::time_point now,
159 aos::monotonic_clock::time_point message_time);
160
Austin Schuh572924a2021-07-30 22:32:12 -0700161 void QueueHeader(
162 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> &&header);
163
Austin Schuhe46492f2021-07-31 19:49:41 -0700164 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader();
165
Austin Schuh58646e22021-08-23 23:51:46 -0700166 monotonic_clock::time_point monotonic_start_time_ = monotonic_clock::min_time;
167
Austin Schuh577610e2021-12-08 12:07:19 -0800168 const Node *node_ = nullptr;
169 size_t node_index_ = 0;
Austin Schuhf5f99f32022-02-07 20:05:37 -0800170 size_t logger_node_index_ = 0;
Austin Schuh572924a2021-07-30 22:32:12 -0700171 LogNamer *log_namer_;
172 UUID parts_uuid_ = UUID::Random();
173 size_t parts_index_ = 0;
174
Austin Schuhb8bca732021-07-30 22:32:00 -0700175 std::function<void(NewDataWriter *)> reopen_;
176 std::function<void(NewDataWriter *)> close_;
Austin Schuh572924a2021-07-30 22:32:12 -0700177 bool header_written_ = false;
Austin Schuhe46492f2021-07-31 19:49:41 -0700178
Austin Schuh72211ae2021-08-05 14:02:30 -0700179 std::vector<State> state_;
Austin Schuh48d10d62022-10-16 22:19:23 -0700180
181 size_t max_message_size_;
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -0700182
183 // Each data writer logs the channels for that node, i.e.
184 // each data writer writes one file. We may encounter messages which
185 // violate the max out of order duration specified in the header of that file.
186 // Rotate the data writer and start a new part for that particular file.
187 // This shouldn't affect the headers of other data writers, so make this
188 // a property of individual data writer instead of the overall log.
189 std::chrono::nanoseconds max_out_of_order_duration_;
190
191 // Monotonic time point of the latest message we've logged so far, i.e
192 // Message X - time Z
193 // Message Y - time Z + 1
194 // newest_message_time_ = Z + 1 (even if X was logged after Y)
195 //
196 // Since the messages can be logged out of order, this helps determine if
197 // max out of order duration was violated.
198 monotonic_clock::time_point newest_message_time_ = monotonic_clock::min_time;
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -0700199
200 // An array with a bool for each value of StoredDataType representing if that
201 // data type is allowed to be logged by this object.
202 std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1>
203 allowed_data_types_;
Austin Schuhb8bca732021-07-30 22:32:00 -0700204};
205
Austin Schuhcb5601b2020-09-10 15:29:59 -0700206// Interface describing how to name, track, and add headers to log file parts.
207class LogNamer {
208 public:
209 // Constructs a LogNamer with the primary node (ie the one the logger runs on)
210 // being node.
Austin Schuh5b728b72021-06-16 14:57:15 -0700211 LogNamer(const aos::Configuration *configuration, EventLoop *event_loop,
212 const aos::Node *node)
Austin Schuha499cea2021-07-31 19:49:53 -0700213 : event_loop_(event_loop),
Austin Schuh5b728b72021-06-16 14:57:15 -0700214 configuration_(configuration),
215 node_(node),
Austin Schuha499cea2021-07-31 19:49:53 -0700216 logger_node_index_(configuration::GetNodeIndex(configuration_, node_)) {
Austin Schuh73340842021-07-30 22:32:06 -0700217 nodes_.emplace_back(node_);
Austin Schuh73340842021-07-30 22:32:06 -0700218 }
Alexei Strots01395492023-03-20 13:59:56 -0700219 virtual ~LogNamer() = default;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700220
Brian Silverman87ac0402020-09-17 14:47:01 -0700221 // Returns a writer for writing data from messages on this channel (on the
222 // primary node).
223 //
224 // The returned pointer will stay valid across rotations, but the object it
225 // points to will be assigned to.
Austin Schuhb8bca732021-07-30 22:32:00 -0700226 virtual NewDataWriter *MakeWriter(const Channel *channel) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700227
Brian Silverman87ac0402020-09-17 14:47:01 -0700228 // Returns a writer for writing timestamps from messages on this channel (on
229 // the primary node).
230 //
231 // The returned pointer will stay valid across rotations, but the object it
232 // points to will be assigned to.
Austin Schuhb8bca732021-07-30 22:32:00 -0700233 virtual NewDataWriter *MakeTimestampWriter(const Channel *channel) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700234
235 // Returns a writer for writing timestamps delivered over the special
236 // /aos/remote_timestamps/* channels. node is the node that the timestamps
Brian Silverman87ac0402020-09-17 14:47:01 -0700237 // are forwarded back from (to the primary node).
238 //
239 // The returned pointer will stay valid across rotations, but the object it
240 // points to will be assigned to.
Austin Schuh73340842021-07-30 22:32:06 -0700241 virtual NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel,
242 const Node *node) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700243
Austin Schuh73340842021-07-30 22:32:06 -0700244 // Rotates all log files for the provided node.
245 virtual void Rotate(const Node *node) = 0;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700246
247 // Returns all the nodes that data is being written for.
248 const std::vector<const Node *> &nodes() const { return nodes_; }
249
Austin Schuh08dba8f2023-05-01 08:29:30 -0700250 // Closes all existing log data writers. No more data may be written after
251 // this.
252 virtual WriteCode Close() = 0;
253
Austin Schuhcb5601b2020-09-10 15:29:59 -0700254 // Returns the node the logger is running on.
255 const Node *node() const { return node_; }
Austin Schuhe46492f2021-07-31 19:49:41 -0700256 const UUID &logger_node_boot_uuid() const { return logger_node_boot_uuid_; }
257 size_t logger_node_index() const { return logger_node_index_; }
Austin Schuhcb5601b2020-09-10 15:29:59 -0700258
Austin Schuh8c399962020-12-25 21:51:45 -0800259 // Writes out the nested Configuration object to the config file location.
260 virtual void WriteConfiguration(
261 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header,
262 std::string_view config_sha256) = 0;
263
Austin Schuh73340842021-07-30 22:32:06 -0700264 void SetHeaderTemplate(
265 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header) {
266 header_ = std::move(header);
Austin Schuhe46492f2021-07-31 19:49:41 -0700267 logger_node_boot_uuid_ =
268 UUID::FromString(header_.message().logger_node_boot_uuid());
Austin Schuh73340842021-07-30 22:32:06 -0700269 }
Austin Schuhcb5601b2020-09-10 15:29:59 -0700270
Austin Schuh60e77942022-05-16 17:48:24 -0700271 void ClearStartTimes() { node_states_.clear(); }
Austin Schuh58646e22021-08-23 23:51:46 -0700272
273 void SetStartTimes(size_t node_index, const UUID &boot_uuid,
Austin Schuh73340842021-07-30 22:32:06 -0700274 monotonic_clock::time_point monotonic_start_time,
275 realtime_clock::time_point realtime_start_time,
276 monotonic_clock::time_point logger_monotonic_start_time,
277 realtime_clock::time_point logger_realtime_start_time) {
Austin Schuh58646e22021-08-23 23:51:46 -0700278 VLOG(1) << "Setting node " << node_index << " to start time "
279 << monotonic_start_time << " rt " << realtime_start_time << " UUID "
280 << boot_uuid;
281 NodeState *node_state = GetNodeState(node_index, boot_uuid);
282 node_state->monotonic_start_time = monotonic_start_time;
283 node_state->realtime_start_time = realtime_start_time;
284 node_state->logger_monotonic_start_time = logger_monotonic_start_time;
285 node_state->logger_realtime_start_time = logger_realtime_start_time;
Austin Schuh73340842021-07-30 22:32:06 -0700286 }
287
Austin Schuh58646e22021-08-23 23:51:46 -0700288 monotonic_clock::time_point monotonic_start_time(size_t node_index,
289 const UUID &boot_uuid) {
290 DCHECK_NE(boot_uuid, UUID::Zero());
291
292 NodeState *node_state = GetNodeState(node_index, boot_uuid);
293 return node_state->monotonic_start_time;
Austin Schuh73340842021-07-30 22:32:06 -0700294 }
295
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -0700296 // This returns the initial out of order duration set in the header template
297 // by the logger based on polling period. It may be different than the actual
298 // duration used by the data writer.
299 std::chrono::nanoseconds base_max_out_of_order_duration() const {
300 return std::chrono::nanoseconds(
301 header_.message().max_out_of_order_duration());
302 }
303
Austin Schuh73340842021-07-30 22:32:06 -0700304 protected:
Austin Schuh73340842021-07-30 22:32:06 -0700305 // Structure with state per node about times and such.
Austin Schuh73340842021-07-30 22:32:06 -0700306 struct NodeState {
307 // Time when this node started logging.
308 monotonic_clock::time_point monotonic_start_time =
309 monotonic_clock::min_time;
310 realtime_clock::time_point realtime_start_time = realtime_clock::min_time;
311
312 // Corresponding time on the logger node when it started logging.
313 monotonic_clock::time_point logger_monotonic_start_time =
314 monotonic_clock::min_time;
315 realtime_clock::time_point logger_realtime_start_time =
316 realtime_clock::min_time;
Austin Schuh73340842021-07-30 22:32:06 -0700317 };
Austin Schuh58646e22021-08-23 23:51:46 -0700318
319 // Creates a new header by copying fields out of the template and combining
320 // them with the arguments provided.
321 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader(
322 size_t node_index, const std::vector<NewDataWriter::State> &state,
Mithun Bharadwaja5cb8e02023-08-02 16:10:40 -0700323 const UUID &parts_uuid, int parts_index,
Mithun Bharadwaja5f9d482023-08-02 16:10:40 -0700324 std::chrono::nanoseconds max_out_of_order_duration,
325 const std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1>
326 &allowed_data_types);
Austin Schuh58646e22021-08-23 23:51:46 -0700327
328 EventLoop *event_loop_;
329 const Configuration *const configuration_;
330 const Node *const node_;
331 const size_t logger_node_index_;
332 UUID logger_node_boot_uuid_;
333 std::vector<const Node *> nodes_;
334
335 friend NewDataWriter;
336
337 // Returns the start/stop time state structure for a node and boot. We can
338 // have data from multiple boots, and it makes sense to reuse the start/stop
339 // times if we get data from the same boot again.
340 NodeState *GetNodeState(size_t node_index, const UUID &boot_uuid);
341
342 absl::btree_map<std::pair<size_t, UUID>, NodeState> node_states_;
Austin Schuh73340842021-07-30 22:32:06 -0700343
344 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header_ =
345 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader>::Empty();
Austin Schuhcb5601b2020-09-10 15:29:59 -0700346};
347
Alexei Strots01395492023-03-20 13:59:56 -0700348// Log namer which uses a config to name a bunch of files.
Austin Schuhcb5601b2020-09-10 15:29:59 -0700349class MultiNodeLogNamer : public LogNamer {
350 public:
Alexei Strotscaf17d32023-04-03 22:31:11 -0700351 MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend,
Alexei Strots01395492023-03-20 13:59:56 -0700352 EventLoop *event_loop);
Alexei Strotscaf17d32023-04-03 22:31:11 -0700353 MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend,
Austin Schuh5b728b72021-06-16 14:57:15 -0700354 const Configuration *configuration, EventLoop *event_loop,
355 const Node *node);
Brian Silvermancb805822020-10-06 17:43:35 -0700356 ~MultiNodeLogNamer() override;
357
Austin Schuh48d10d62022-10-16 22:19:23 -0700358 // Sets the function for creating encoders. The argument is the max message
359 // size (including headers) that will be written into this encoder.
Brian Silvermancb805822020-10-06 17:43:35 -0700360 //
361 // Defaults to just creating DummyEncoders.
362 void set_encoder_factory(
Austin Schuh48d10d62022-10-16 22:19:23 -0700363 std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory) {
Brian Silvermancb805822020-10-06 17:43:35 -0700364 encoder_factory_ = std::move(encoder_factory);
365 }
366
367 // Sets an additional file extension.
368 //
369 // Defaults to nothing.
370 void set_extension(std::string_view extension) { extension_ = extension; }
Brian Silverman1f345222020-09-24 21:14:48 -0700371
Brian Silvermana621f522020-09-30 16:52:43 -0700372 // A list of all the filenames we've written.
373 //
374 // This only includes the part after base_name().
375 const std::vector<std::string> &all_filenames() const {
376 return all_filenames_;
377 }
378
Austin Schuh73340842021-07-30 22:32:06 -0700379 void Rotate(const Node *node) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700380
Austin Schuh8c399962020-12-25 21:51:45 -0800381 void WriteConfiguration(
382 aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header,
383 std::string_view config_sha256) override;
384
Austin Schuhb8bca732021-07-30 22:32:00 -0700385 NewDataWriter *MakeWriter(const Channel *channel) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700386
Austin Schuhb8bca732021-07-30 22:32:00 -0700387 NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel,
Austin Schuh73340842021-07-30 22:32:06 -0700388 const Node *node) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700389
Austin Schuhb8bca732021-07-30 22:32:00 -0700390 NewDataWriter *MakeTimestampWriter(const Channel *channel) override;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700391
Brian Silverman0465fcf2020-09-24 00:29:18 -0700392 // Indicates that at least one file ran out of space. Once this happens, we
393 // stop trying to open new files, to avoid writing any files with holes from
394 // previous parts.
395 //
396 // Besides this function, this object will silently stop logging data when
397 // this occurs. If you want to ensure log files are complete, you must call
398 // this method.
Brian Silvermana9f2ec92020-10-06 18:00:53 -0700399 bool ran_out_of_space() const {
400 return accumulate_data_writers<bool>(
Austin Schuhb8bca732021-07-30 22:32:00 -0700401 ran_out_of_space_, [](bool x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600402 CHECK_NOTNULL(data_writer.writer);
Brian Silvermana9f2ec92020-10-06 18:00:53 -0700403 return x ||
404 (data_writer.writer && data_writer.writer->ran_out_of_space());
405 });
406 }
Brian Silverman0465fcf2020-09-24 00:29:18 -0700407
Brian Silverman1f345222020-09-24 21:14:48 -0700408 // Returns the maximum total_bytes() value for all existing
409 // DetachedBufferWriters.
410 //
411 // Returns 0 if no files are open.
412 size_t maximum_total_bytes() const {
Brian Silvermancb805822020-10-06 17:43:35 -0700413 return accumulate_data_writers<size_t>(
Austin Schuhb8bca732021-07-30 22:32:00 -0700414 0, [](size_t x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600415 CHECK_NOTNULL(data_writer.writer);
Brian Silvermancb805822020-10-06 17:43:35 -0700416 return std::max(x, data_writer.writer->total_bytes());
417 });
Brian Silverman1f345222020-09-24 21:14:48 -0700418 }
419
Brian Silverman0465fcf2020-09-24 00:29:18 -0700420 // Closes all existing log files. No more data may be written after this.
421 //
422 // This may set ran_out_of_space().
Austin Schuh08dba8f2023-05-01 08:29:30 -0700423 WriteCode Close() override;
Brian Silverman0465fcf2020-09-24 00:29:18 -0700424
Brian Silvermancb805822020-10-06 17:43:35 -0700425 // Accessors for various statistics. See the identically-named methods in
426 // DetachedBufferWriter for documentation. These are aggregated across all
427 // past and present DetachedBufferWriters.
428 std::chrono::nanoseconds max_write_time() const {
429 return accumulate_data_writers(
430 max_write_time_,
Austin Schuhb8bca732021-07-30 22:32:00 -0700431 [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600432 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700433 return std::max(
434 x, data_writer.writer->WriteStatistics()->max_write_time());
Brian Silvermancb805822020-10-06 17:43:35 -0700435 });
436 }
437 int max_write_time_bytes() const {
438 return std::get<0>(accumulate_data_writers(
439 std::make_tuple(max_write_time_bytes_, max_write_time_),
440 [](std::tuple<int, std::chrono::nanoseconds> x,
Austin Schuhb8bca732021-07-30 22:32:00 -0700441 const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600442 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700443 if (data_writer.writer->WriteStatistics()->max_write_time() >
444 std::get<1>(x)) {
445 return std::make_tuple(
446 data_writer.writer->WriteStatistics()->max_write_time_bytes(),
447 data_writer.writer->WriteStatistics()->max_write_time());
Brian Silvermancb805822020-10-06 17:43:35 -0700448 }
449 return x;
450 }));
451 }
452 int max_write_time_messages() const {
453 return std::get<0>(accumulate_data_writers(
454 std::make_tuple(max_write_time_messages_, max_write_time_),
455 [](std::tuple<int, std::chrono::nanoseconds> x,
Austin Schuhb8bca732021-07-30 22:32:00 -0700456 const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600457 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700458 if (data_writer.writer->WriteStatistics()->max_write_time() >
459 std::get<1>(x)) {
Brian Silvermancb805822020-10-06 17:43:35 -0700460 return std::make_tuple(
Alexei Strots01395492023-03-20 13:59:56 -0700461 data_writer.writer->WriteStatistics()
462 ->max_write_time_messages(),
463 data_writer.writer->WriteStatistics()->max_write_time());
Brian Silvermancb805822020-10-06 17:43:35 -0700464 }
465 return x;
466 }));
467 }
468 std::chrono::nanoseconds total_write_time() const {
469 return accumulate_data_writers(
470 total_write_time_,
Austin Schuhb8bca732021-07-30 22:32:00 -0700471 [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600472 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700473 return x + data_writer.writer->WriteStatistics()->total_write_time();
Brian Silvermancb805822020-10-06 17:43:35 -0700474 });
475 }
476 int total_write_count() const {
477 return accumulate_data_writers(
Austin Schuhb8bca732021-07-30 22:32:00 -0700478 total_write_count_, [](int x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600479 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700480 return x + data_writer.writer->WriteStatistics()->total_write_count();
Brian Silvermancb805822020-10-06 17:43:35 -0700481 });
482 }
483 int total_write_messages() const {
484 return accumulate_data_writers(
Austin Schuhb8bca732021-07-30 22:32:00 -0700485 total_write_messages_, [](int x, const NewDataWriter &data_writer) {
Alexei Strots01395492023-03-20 13:59:56 -0700486 return x +
487 data_writer.writer->WriteStatistics()->total_write_messages();
Brian Silvermancb805822020-10-06 17:43:35 -0700488 });
489 }
490 int total_write_bytes() const {
491 return accumulate_data_writers(
Austin Schuhb8bca732021-07-30 22:32:00 -0700492 total_write_bytes_, [](int x, const NewDataWriter &data_writer) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600493 CHECK_NOTNULL(data_writer.writer);
Alexei Strots01395492023-03-20 13:59:56 -0700494 return x + data_writer.writer->WriteStatistics()->total_write_bytes();
Brian Silvermancb805822020-10-06 17:43:35 -0700495 });
496 }
497
498 void ResetStatistics();
499
Alexei Strotscaf17d32023-04-03 22:31:11 -0700500 protected:
501 // TODO (Alexei): consider to move ownership of log_namer to concrete sub
502 // class and make log_backend_ raw pointer.
503 LogBackend *log_backend() { return log_backend_.get(); }
504 const LogBackend *log_backend() const { return log_backend_.get(); }
505
Austin Schuh6ecfe902023-08-04 22:44:37 -0700506 // Returns the data writer or timestamp writer if we find one for the provided
507 // node.
508 NewDataWriter *FindNodeDataWriter(const Node *node, size_t max_message_size);
509 NewDataWriter *FindNodeTimestampWriter(const Node *node,
510 size_t max_message_size);
511
512 // Saves the data writer or timestamp writer for the provided node.
513 NewDataWriter *AddNodeDataWriter(const Node *node, NewDataWriter &&writer);
514 NewDataWriter *AddNodeTimestampWriter(const Node *node,
515 NewDataWriter &&writer);
516
517 void CloseWriter(std::unique_ptr<DetachedBufferWriter> *writer_pointer);
518
519 void CreateBufferWriter(std::string_view path, size_t max_message_size,
520 std::unique_ptr<DetachedBufferWriter> *destination);
521
522 std::string extension_;
523
Austin Schuhcb5601b2020-09-10 15:29:59 -0700524 private:
Austin Schuhcb5601b2020-09-10 15:29:59 -0700525 // Opens up a writer for timestamps forwarded back.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700526 void OpenForwardedTimestampWriter(const Node *source_node,
Austin Schuhb8bca732021-07-30 22:32:00 -0700527 NewDataWriter *data_writer);
Austin Schuhcb5601b2020-09-10 15:29:59 -0700528
529 // Opens up a writer for remote data.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700530 void OpenDataWriter(const Node *source_node, NewDataWriter *data_writer);
531 void OpenTimestampWriter(NewDataWriter *data_writer);
Austin Schuhcb5601b2020-09-10 15:29:59 -0700532
Austin Schuh6ecfe902023-08-04 22:44:37 -0700533 // Tracks the node in nodes_.
534 void NoticeNode(const Node *source_node);
Austin Schuhcb5601b2020-09-10 15:29:59 -0700535
Brian Silvermancb805822020-10-06 17:43:35 -0700536 // A version of std::accumulate which operates over all of our DataWriters.
537 template <typename T, typename BinaryOperation>
538 T accumulate_data_writers(T t, BinaryOperation op) const {
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700539 for (const std::pair<const Node *const, NewDataWriter> &data_writer :
540 node_data_writers_) {
Maxwell Gumley8ad77782023-07-11 13:27:03 -0600541 if (data_writer.second.writer != nullptr) {
542 t = op(std::move(t), data_writer.second);
543 }
Brian Silvermancb805822020-10-06 17:43:35 -0700544 }
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700545 for (const std::pair<const Node *const, NewDataWriter> &data_writer :
546 node_timestamp_writers_) {
547 if (data_writer.second.writer != nullptr) {
548 t = op(std::move(t), data_writer.second);
549 }
Brian Silvermancb805822020-10-06 17:43:35 -0700550 }
551 return t;
552 }
553
Alexei Strotscaf17d32023-04-03 22:31:11 -0700554 std::unique_ptr<LogBackend> log_backend_;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700555
Brian Silverman0465fcf2020-09-24 00:29:18 -0700556 bool ran_out_of_space_ = false;
Brian Silvermana621f522020-09-30 16:52:43 -0700557 std::vector<std::string> all_filenames_;
Brian Silverman0465fcf2020-09-24 00:29:18 -0700558
Austin Schuh8bdfc492023-02-11 12:53:13 -0800559 std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory_;
Brian Silvermancb805822020-10-06 17:43:35 -0700560
561 // Storage for statistics from previously-rotated DetachedBufferWriters.
562 std::chrono::nanoseconds max_write_time_ = std::chrono::nanoseconds::zero();
563 int max_write_time_bytes_ = -1;
564 int max_write_time_messages_ = -1;
565 std::chrono::nanoseconds total_write_time_ = std::chrono::nanoseconds::zero();
566 int total_write_count_ = 0;
567 int total_write_messages_ = 0;
568 int total_write_bytes_ = 0;
569
Mithun Bharadwaj0c629932023-08-02 16:10:40 -0700570 // Data writer per remote node.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700571 std::map<const Node *, NewDataWriter> node_data_writers_;
Mithun Bharadwaj99aec9e2023-08-02 16:10:40 -0700572 // Remote timestamp writers per node.
Mithun Bharadwajc54aa022023-08-02 16:10:41 -0700573 std::map<const Node *, NewDataWriter> node_timestamp_writers_;
Austin Schuhcb5601b2020-09-10 15:29:59 -0700574};
575
Alexei Strots01395492023-03-20 13:59:56 -0700576// This is specialized log namer that deals with directory centric log events.
577class MultiNodeFilesLogNamer : public MultiNodeLogNamer {
578 public:
579 MultiNodeFilesLogNamer(std::string_view base_name, EventLoop *event_loop)
colleen61276dc2023-06-01 09:23:29 -0700580 : MultiNodeLogNamer(
581 std::make_unique<RenamableFileBackend>(base_name, false),
582 event_loop) {}
Alexei Strots01395492023-03-20 13:59:56 -0700583
584 MultiNodeFilesLogNamer(std::string_view base_name,
585 const Configuration *configuration,
586 EventLoop *event_loop, const Node *node)
colleen61276dc2023-06-01 09:23:29 -0700587 : MultiNodeLogNamer(
588 std::make_unique<RenamableFileBackend>(base_name, false),
589 configuration, event_loop, node) {}
590
591 MultiNodeFilesLogNamer(EventLoop *event_loop,
592 std::unique_ptr<RenamableFileBackend> backend)
593 : MultiNodeLogNamer(std::move(backend), event_loop) {}
594
Alexei Strots01395492023-03-20 13:59:56 -0700595 ~MultiNodeFilesLogNamer() override = default;
Alexei Strotscaf17d32023-04-03 22:31:11 -0700596
597 std::string_view base_name() const {
598 return renamable_file_backend()->base_name();
599 }
600
601 // Rotate should be called at least once in between calls to set_base_name.
602 // Otherwise, temporary files will not be recoverable.
603 // Rotate is called by Logger::RenameLogBase, which is currently the only user
604 // of this method.
605 // Only renaming the folder is supported, not the file base name.
606 void set_base_name(std::string_view base_name) {
607 renamable_file_backend()->RenameLogBase(base_name);
608 }
609
610 // When enabled, this will write files under names beginning
611 // with the .tmp suffix, and then rename them to the desired name after
612 // they are fully written.
613 //
614 // This is useful to enable incremental copying of the log files.
615 //
616 // Defaults to writing directly to the final filename.
617 void EnableTempFiles() { renamable_file_backend()->EnableTempFiles(); }
618
619 private:
620 RenamableFileBackend *renamable_file_backend() {
621 return reinterpret_cast<RenamableFileBackend *>(log_backend());
622 }
623 const RenamableFileBackend *renamable_file_backend() const {
624 return reinterpret_cast<const RenamableFileBackend *>(log_backend());
625 }
Alexei Strots01395492023-03-20 13:59:56 -0700626};
627
Austin Schuh6ecfe902023-08-04 22:44:37 -0700628// Class which dumps all data from each node into a single file per node. This
629// is mostly interesting for testing.
630class MinimalFileMultiNodeLogNamer : public MultiNodeFilesLogNamer {
631 public:
632 MinimalFileMultiNodeLogNamer(std::string_view base_name,
633 EventLoop *event_loop)
634 : MultiNodeFilesLogNamer(base_name, event_loop) {}
635 MinimalFileMultiNodeLogNamer(std::string_view base_name,
636 const Configuration *configuration,
637 EventLoop *event_loop, const Node *node)
638 : MultiNodeFilesLogNamer(base_name, configuration, event_loop, node) {}
639
640 NewDataWriter *MakeWriter(const Channel *channel) override;
641
642 NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel,
643 const Node *node) override;
644
645 NewDataWriter *MakeTimestampWriter(const Channel *channel) override;
646
647 private:
648 // Names the data writer.
649 void OpenNodeWriter(const Node *source_node, NewDataWriter *data_writer);
650};
651
Stephan Pleinesd99b1ee2024-02-02 20:56:44 -0800652} // namespace aos::logger
Austin Schuhcb5601b2020-09-10 15:29:59 -0700653
654#endif // AOS_EVENTS_LOGGING_LOG_NAMER_H_