Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 1 | #ifndef AOS_EVENTS_LOGGING_LOG_NAMER_H_ |
| 2 | #define AOS_EVENTS_LOGGING_LOG_NAMER_H_ |
| 3 | |
| 4 | #include <functional> |
| 5 | #include <map> |
| 6 | #include <memory> |
| 7 | #include <string_view> |
| 8 | #include <vector> |
| 9 | |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 10 | #include "absl/container/btree_map.h" |
Philipp Schrader | 790cb54 | 2023-07-05 21:06:52 -0700 | [diff] [blame] | 11 | #include "flatbuffers/flatbuffers.h" |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 12 | #include "glog/logging.h" |
Philipp Schrader | 790cb54 | 2023-07-05 21:06:52 -0700 | [diff] [blame] | 13 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 14 | #include "aos/events/logging/logfile_utils.h" |
| 15 | #include "aos/events/logging/logger_generated.h" |
Austin Schuh | 4385b14 | 2021-03-14 21:31:13 -0700 | [diff] [blame] | 16 | #include "aos/uuid.h" |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 17 | |
Stephan Pleines | d99b1ee | 2024-02-02 20:56:44 -0800 | [diff] [blame] | 18 | namespace aos::logger { |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 19 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 20 | class LogNamer; |
| 21 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 22 | // TODO(austin): Rename this back to DataWriter once all other callers are of |
| 23 | // the old DataWriter. |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 24 | // |
| 25 | // Class to manage writing data to log files. This lets us track which boot the |
| 26 | // written header has in it, and if the header has been written or not. |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 27 | // |
| 28 | // The design of this class is that instead of being notified when any of the |
| 29 | // header data changes, it polls and owns that decision. This makes it much |
| 30 | // harder to write corrupted data. If that becomes a performance problem, we |
| 31 | // can DCHECK and take it out of production binaries. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 32 | class NewDataWriter { |
| 33 | public: |
| 34 | // Constructs a NewDataWriter. |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 35 | // log_namer is the log namer which holds the config and any other data we |
| 36 | // need for our header. |
| 37 | // node is the node whom's prespective we are logging from. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 38 | // reopen is called whenever a file needs to be reopened. |
| 39 | // close is called to close that file and extract any statistics. |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 40 | NewDataWriter(LogNamer *log_namer, const Node *node, const Node *logger_node, |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 41 | std::function<void(NewDataWriter *)> reopen, |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 42 | std::function<void(NewDataWriter *)> close, |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 43 | size_t max_message_size, |
| 44 | std::initializer_list<StoredDataType> types); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 45 | |
| 46 | void UpdateMaxMessageSize(size_t new_size) { |
| 47 | if (new_size > max_message_size_) { |
Alexei Strots | bc082d8 | 2023-05-03 08:43:42 -0700 | [diff] [blame] | 48 | CHECK(!header_written_) << ": Tried to update to " << new_size << ", was " |
| 49 | << max_message_size_ << " for " << name(); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 50 | max_message_size_ = new_size; |
| 51 | } |
| 52 | } |
| 53 | size_t max_message_size() const { return max_message_size_; } |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 54 | |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 55 | std::chrono::nanoseconds max_out_of_order_duration() const { |
| 56 | return max_out_of_order_duration_; |
| 57 | } |
| 58 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 59 | NewDataWriter(NewDataWriter &&other) = default; |
| 60 | aos::logger::NewDataWriter &operator=(NewDataWriter &&other) = default; |
| 61 | NewDataWriter(const NewDataWriter &) = delete; |
| 62 | void operator=(const NewDataWriter &) = delete; |
| 63 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 64 | ~NewDataWriter(); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 65 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 66 | // Rotates the log file, delaying writing the new header until data arrives. |
| 67 | void Rotate(); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 68 | |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 69 | // Updates all the metadata in the log file about the remote node which this |
| 70 | // message is from. |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 71 | void UpdateRemote(size_t remote_node_index, const UUID &remote_node_boot_uuid, |
| 72 | monotonic_clock::time_point monotonic_remote_time, |
| 73 | monotonic_clock::time_point monotonic_event_time, |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 74 | bool reliable, |
| 75 | monotonic_clock::time_point monotonic_timestamp_time = |
| 76 | monotonic_clock::min_time); |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 77 | |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 78 | // Coppies a message with the provided boot UUID. |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 79 | void CopyDataMessage(DataEncoder::Copier *copier, |
| 80 | const UUID &source_node_boot_uuid, |
| 81 | aos::monotonic_clock::time_point now, |
| 82 | aos::monotonic_clock::time_point message_time); |
| 83 | void CopyTimestampMessage(DataEncoder::Copier *copier, |
| 84 | const UUID &source_node_boot_uuid, |
| 85 | aos::monotonic_clock::time_point now, |
| 86 | aos::monotonic_clock::time_point message_time); |
| 87 | void CopyRemoteTimestampMessage( |
| 88 | DataEncoder::Copier *copier, const UUID &source_node_boot_uuid, |
| 89 | aos::monotonic_clock::time_point now, |
| 90 | aos::monotonic_clock::time_point message_time); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 91 | |
Austin Schuh | 5e14d84 | 2022-01-21 12:02:15 -0800 | [diff] [blame] | 92 | // Updates the current boot for the source node. This is useful when you want |
| 93 | // to queue a message that may trigger a reboot rotation, but then need to |
| 94 | // update the remote timestamps. |
| 95 | void UpdateBoot(const UUID &source_node_boot_uuid); |
| 96 | |
Alexei Strots | bc082d8 | 2023-05-03 08:43:42 -0700 | [diff] [blame] | 97 | // Returns the name of the writer. It may be a filename, but assume it is not. |
| 98 | std::string_view name() const { return writer ? writer->name() : "(closed)"; } |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 99 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 100 | void Close(); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 101 | |
| 102 | std::unique_ptr<DetachedBufferWriter> writer = nullptr; |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 103 | |
| 104 | size_t node_index() const { return node_index_; } |
| 105 | const UUID &parts_uuid() const { return parts_uuid_; } |
| 106 | size_t parts_index() const { return parts_index_; } |
| 107 | const Node *node() const { return node_; } |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 108 | |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 109 | // Datastructure used to capture all the information about a remote node. |
| 110 | struct State { |
| 111 | // Boot UUID of the node. |
| 112 | UUID boot_uuid = UUID::Zero(); |
| 113 | // Timestamp on the remote monotonic clock of the oldest message sent to |
| 114 | // node_index_. |
| 115 | monotonic_clock::time_point oldest_remote_monotonic_timestamp = |
| 116 | monotonic_clock::max_time; |
| 117 | // Timestamp on the local monotonic clock of the message in |
| 118 | // oldest_remote_monotonic_timestamp. |
| 119 | monotonic_clock::time_point oldest_local_monotonic_timestamp = |
| 120 | monotonic_clock::max_time; |
| 121 | // Timestamp on the remote monotonic clock of the oldest message sent to |
| 122 | // node_index_, excluding messages forwarded with time_to_live() == 0. |
| 123 | monotonic_clock::time_point oldest_remote_unreliable_monotonic_timestamp = |
| 124 | monotonic_clock::max_time; |
| 125 | // Timestamp on the local monotonic clock of the message in |
| 126 | // oldest_local_unreliable_monotonic_timestamp. |
| 127 | monotonic_clock::time_point oldest_local_unreliable_monotonic_timestamp = |
| 128 | monotonic_clock::max_time; |
Austin Schuh | bfe6c57 | 2022-01-27 20:48:20 -0800 | [diff] [blame] | 129 | |
| 130 | // Timestamp on the remote monotonic clock of the oldest message sent to |
| 131 | // node_index_, only including messages forwarded with time_to_live() == 0. |
| 132 | monotonic_clock::time_point oldest_remote_reliable_monotonic_timestamp = |
| 133 | monotonic_clock::max_time; |
| 134 | // Timestamp on the local monotonic clock of the message in |
| 135 | // oldest_local_reliable_monotonic_timestamp. |
| 136 | monotonic_clock::time_point oldest_local_reliable_monotonic_timestamp = |
| 137 | monotonic_clock::max_time; |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 138 | |
| 139 | // Timestamp on the remote monotonic clock of the oldest message timestamp |
| 140 | // sent back to logger_node_index_. The remote here will be the node this |
| 141 | // part is from the perspective of, ie node_index_. |
| 142 | monotonic_clock::time_point |
| 143 | oldest_logger_remote_unreliable_monotonic_timestamp = |
| 144 | monotonic_clock::max_time; |
| 145 | // The time on the monotonic clock of the logger when this timestamp made it |
| 146 | // back to the logger (logger_node_index_). |
| 147 | monotonic_clock::time_point |
| 148 | oldest_logger_local_unreliable_monotonic_timestamp = |
| 149 | monotonic_clock::max_time; |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 150 | }; |
| 151 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 152 | private: |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 153 | // Signals that a node has rebooted. |
Austin Schuh | 5e14d84 | 2022-01-21 12:02:15 -0800 | [diff] [blame] | 154 | void Reboot(const UUID &source_node_boot_uuid); |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 155 | |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 156 | void CopyMessage(DataEncoder::Copier *copier, |
| 157 | const UUID &source_node_boot_uuid, |
| 158 | aos::monotonic_clock::time_point now, |
| 159 | aos::monotonic_clock::time_point message_time); |
| 160 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 161 | void QueueHeader( |
| 162 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> &&header); |
| 163 | |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 164 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader(); |
| 165 | |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 166 | monotonic_clock::time_point monotonic_start_time_ = monotonic_clock::min_time; |
| 167 | |
Austin Schuh | 577610e | 2021-12-08 12:07:19 -0800 | [diff] [blame] | 168 | const Node *node_ = nullptr; |
| 169 | size_t node_index_ = 0; |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 170 | size_t logger_node_index_ = 0; |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 171 | LogNamer *log_namer_; |
| 172 | UUID parts_uuid_ = UUID::Random(); |
| 173 | size_t parts_index_ = 0; |
| 174 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 175 | std::function<void(NewDataWriter *)> reopen_; |
| 176 | std::function<void(NewDataWriter *)> close_; |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 177 | bool header_written_ = false; |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 178 | |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 179 | std::vector<State> state_; |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 180 | |
| 181 | size_t max_message_size_; |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 182 | |
| 183 | // Each data writer logs the channels for that node, i.e. |
| 184 | // each data writer writes one file. We may encounter messages which |
| 185 | // violate the max out of order duration specified in the header of that file. |
| 186 | // Rotate the data writer and start a new part for that particular file. |
| 187 | // This shouldn't affect the headers of other data writers, so make this |
| 188 | // a property of individual data writer instead of the overall log. |
| 189 | std::chrono::nanoseconds max_out_of_order_duration_; |
| 190 | |
| 191 | // Monotonic time point of the latest message we've logged so far, i.e |
| 192 | // Message X - time Z |
| 193 | // Message Y - time Z + 1 |
| 194 | // newest_message_time_ = Z + 1 (even if X was logged after Y) |
| 195 | // |
| 196 | // Since the messages can be logged out of order, this helps determine if |
| 197 | // max out of order duration was violated. |
| 198 | monotonic_clock::time_point newest_message_time_ = monotonic_clock::min_time; |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 199 | |
| 200 | // An array with a bool for each value of StoredDataType representing if that |
| 201 | // data type is allowed to be logged by this object. |
| 202 | std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1> |
| 203 | allowed_data_types_; |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 204 | }; |
| 205 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 206 | // Interface describing how to name, track, and add headers to log file parts. |
| 207 | class LogNamer { |
| 208 | public: |
| 209 | // Constructs a LogNamer with the primary node (ie the one the logger runs on) |
| 210 | // being node. |
Austin Schuh | 5b728b7 | 2021-06-16 14:57:15 -0700 | [diff] [blame] | 211 | LogNamer(const aos::Configuration *configuration, EventLoop *event_loop, |
| 212 | const aos::Node *node) |
Austin Schuh | a499cea | 2021-07-31 19:49:53 -0700 | [diff] [blame] | 213 | : event_loop_(event_loop), |
Austin Schuh | 5b728b7 | 2021-06-16 14:57:15 -0700 | [diff] [blame] | 214 | configuration_(configuration), |
| 215 | node_(node), |
Austin Schuh | a499cea | 2021-07-31 19:49:53 -0700 | [diff] [blame] | 216 | logger_node_index_(configuration::GetNodeIndex(configuration_, node_)) { |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 217 | nodes_.emplace_back(node_); |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 218 | } |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 219 | virtual ~LogNamer() = default; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 220 | |
Brian Silverman | 87ac040 | 2020-09-17 14:47:01 -0700 | [diff] [blame] | 221 | // Returns a writer for writing data from messages on this channel (on the |
| 222 | // primary node). |
| 223 | // |
| 224 | // The returned pointer will stay valid across rotations, but the object it |
| 225 | // points to will be assigned to. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 226 | virtual NewDataWriter *MakeWriter(const Channel *channel) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 227 | |
Brian Silverman | 87ac040 | 2020-09-17 14:47:01 -0700 | [diff] [blame] | 228 | // Returns a writer for writing timestamps from messages on this channel (on |
| 229 | // the primary node). |
| 230 | // |
| 231 | // The returned pointer will stay valid across rotations, but the object it |
| 232 | // points to will be assigned to. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 233 | virtual NewDataWriter *MakeTimestampWriter(const Channel *channel) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 234 | |
| 235 | // Returns a writer for writing timestamps delivered over the special |
| 236 | // /aos/remote_timestamps/* channels. node is the node that the timestamps |
Brian Silverman | 87ac040 | 2020-09-17 14:47:01 -0700 | [diff] [blame] | 237 | // are forwarded back from (to the primary node). |
| 238 | // |
| 239 | // The returned pointer will stay valid across rotations, but the object it |
| 240 | // points to will be assigned to. |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 241 | virtual NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel, |
| 242 | const Node *node) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 243 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 244 | // Rotates all log files for the provided node. |
| 245 | virtual void Rotate(const Node *node) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 246 | |
| 247 | // Returns all the nodes that data is being written for. |
| 248 | const std::vector<const Node *> &nodes() const { return nodes_; } |
| 249 | |
Austin Schuh | 08dba8f | 2023-05-01 08:29:30 -0700 | [diff] [blame] | 250 | // Closes all existing log data writers. No more data may be written after |
| 251 | // this. |
| 252 | virtual WriteCode Close() = 0; |
| 253 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 254 | // Returns the node the logger is running on. |
| 255 | const Node *node() const { return node_; } |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 256 | const UUID &logger_node_boot_uuid() const { return logger_node_boot_uuid_; } |
| 257 | size_t logger_node_index() const { return logger_node_index_; } |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 258 | |
Austin Schuh | 8c39996 | 2020-12-25 21:51:45 -0800 | [diff] [blame] | 259 | // Writes out the nested Configuration object to the config file location. |
| 260 | virtual void WriteConfiguration( |
| 261 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header, |
| 262 | std::string_view config_sha256) = 0; |
| 263 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 264 | void SetHeaderTemplate( |
| 265 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header) { |
| 266 | header_ = std::move(header); |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 267 | logger_node_boot_uuid_ = |
| 268 | UUID::FromString(header_.message().logger_node_boot_uuid()); |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 269 | } |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 270 | |
Austin Schuh | 60e7794 | 2022-05-16 17:48:24 -0700 | [diff] [blame] | 271 | void ClearStartTimes() { node_states_.clear(); } |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 272 | |
| 273 | void SetStartTimes(size_t node_index, const UUID &boot_uuid, |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 274 | monotonic_clock::time_point monotonic_start_time, |
| 275 | realtime_clock::time_point realtime_start_time, |
| 276 | monotonic_clock::time_point logger_monotonic_start_time, |
| 277 | realtime_clock::time_point logger_realtime_start_time) { |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 278 | VLOG(1) << "Setting node " << node_index << " to start time " |
| 279 | << monotonic_start_time << " rt " << realtime_start_time << " UUID " |
| 280 | << boot_uuid; |
| 281 | NodeState *node_state = GetNodeState(node_index, boot_uuid); |
| 282 | node_state->monotonic_start_time = monotonic_start_time; |
| 283 | node_state->realtime_start_time = realtime_start_time; |
| 284 | node_state->logger_monotonic_start_time = logger_monotonic_start_time; |
| 285 | node_state->logger_realtime_start_time = logger_realtime_start_time; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 286 | } |
| 287 | |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 288 | monotonic_clock::time_point monotonic_start_time(size_t node_index, |
| 289 | const UUID &boot_uuid) { |
| 290 | DCHECK_NE(boot_uuid, UUID::Zero()); |
| 291 | |
| 292 | NodeState *node_state = GetNodeState(node_index, boot_uuid); |
| 293 | return node_state->monotonic_start_time; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 294 | } |
| 295 | |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 296 | // This returns the initial out of order duration set in the header template |
| 297 | // by the logger based on polling period. It may be different than the actual |
| 298 | // duration used by the data writer. |
| 299 | std::chrono::nanoseconds base_max_out_of_order_duration() const { |
| 300 | return std::chrono::nanoseconds( |
| 301 | header_.message().max_out_of_order_duration()); |
| 302 | } |
| 303 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 304 | protected: |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 305 | // Structure with state per node about times and such. |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 306 | struct NodeState { |
| 307 | // Time when this node started logging. |
| 308 | monotonic_clock::time_point monotonic_start_time = |
| 309 | monotonic_clock::min_time; |
| 310 | realtime_clock::time_point realtime_start_time = realtime_clock::min_time; |
| 311 | |
| 312 | // Corresponding time on the logger node when it started logging. |
| 313 | monotonic_clock::time_point logger_monotonic_start_time = |
| 314 | monotonic_clock::min_time; |
| 315 | realtime_clock::time_point logger_realtime_start_time = |
| 316 | realtime_clock::min_time; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 317 | }; |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 318 | |
| 319 | // Creates a new header by copying fields out of the template and combining |
| 320 | // them with the arguments provided. |
| 321 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader( |
| 322 | size_t node_index, const std::vector<NewDataWriter::State> &state, |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 323 | const UUID &parts_uuid, int parts_index, |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 324 | std::chrono::nanoseconds max_out_of_order_duration, |
| 325 | const std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1> |
| 326 | &allowed_data_types); |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 327 | |
| 328 | EventLoop *event_loop_; |
| 329 | const Configuration *const configuration_; |
| 330 | const Node *const node_; |
| 331 | const size_t logger_node_index_; |
| 332 | UUID logger_node_boot_uuid_; |
| 333 | std::vector<const Node *> nodes_; |
| 334 | |
| 335 | friend NewDataWriter; |
| 336 | |
| 337 | // Returns the start/stop time state structure for a node and boot. We can |
| 338 | // have data from multiple boots, and it makes sense to reuse the start/stop |
| 339 | // times if we get data from the same boot again. |
| 340 | NodeState *GetNodeState(size_t node_index, const UUID &boot_uuid); |
| 341 | |
| 342 | absl::btree_map<std::pair<size_t, UUID>, NodeState> node_states_; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 343 | |
| 344 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header_ = |
| 345 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader>::Empty(); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 346 | }; |
| 347 | |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 348 | // Log namer which uses a config to name a bunch of files. |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 349 | class MultiNodeLogNamer : public LogNamer { |
| 350 | public: |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 351 | MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend, |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 352 | EventLoop *event_loop); |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 353 | MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend, |
Austin Schuh | 5b728b7 | 2021-06-16 14:57:15 -0700 | [diff] [blame] | 354 | const Configuration *configuration, EventLoop *event_loop, |
| 355 | const Node *node); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 356 | ~MultiNodeLogNamer() override; |
| 357 | |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 358 | // Sets the function for creating encoders. The argument is the max message |
| 359 | // size (including headers) that will be written into this encoder. |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 360 | // |
| 361 | // Defaults to just creating DummyEncoders. |
| 362 | void set_encoder_factory( |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 363 | std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory) { |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 364 | encoder_factory_ = std::move(encoder_factory); |
| 365 | } |
| 366 | |
| 367 | // Sets an additional file extension. |
| 368 | // |
| 369 | // Defaults to nothing. |
| 370 | void set_extension(std::string_view extension) { extension_ = extension; } |
Brian Silverman | 1f34522 | 2020-09-24 21:14:48 -0700 | [diff] [blame] | 371 | |
Brian Silverman | a621f52 | 2020-09-30 16:52:43 -0700 | [diff] [blame] | 372 | // A list of all the filenames we've written. |
| 373 | // |
| 374 | // This only includes the part after base_name(). |
| 375 | const std::vector<std::string> &all_filenames() const { |
| 376 | return all_filenames_; |
| 377 | } |
| 378 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 379 | void Rotate(const Node *node) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 380 | |
Austin Schuh | 8c39996 | 2020-12-25 21:51:45 -0800 | [diff] [blame] | 381 | void WriteConfiguration( |
| 382 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header, |
| 383 | std::string_view config_sha256) override; |
| 384 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 385 | NewDataWriter *MakeWriter(const Channel *channel) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 386 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 387 | NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel, |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 388 | const Node *node) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 389 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 390 | NewDataWriter *MakeTimestampWriter(const Channel *channel) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 391 | |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 392 | // Indicates that at least one file ran out of space. Once this happens, we |
| 393 | // stop trying to open new files, to avoid writing any files with holes from |
| 394 | // previous parts. |
| 395 | // |
| 396 | // Besides this function, this object will silently stop logging data when |
| 397 | // this occurs. If you want to ensure log files are complete, you must call |
| 398 | // this method. |
Brian Silverman | a9f2ec9 | 2020-10-06 18:00:53 -0700 | [diff] [blame] | 399 | bool ran_out_of_space() const { |
| 400 | return accumulate_data_writers<bool>( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 401 | ran_out_of_space_, [](bool x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 402 | CHECK_NOTNULL(data_writer.writer); |
Brian Silverman | a9f2ec9 | 2020-10-06 18:00:53 -0700 | [diff] [blame] | 403 | return x || |
| 404 | (data_writer.writer && data_writer.writer->ran_out_of_space()); |
| 405 | }); |
| 406 | } |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 407 | |
Brian Silverman | 1f34522 | 2020-09-24 21:14:48 -0700 | [diff] [blame] | 408 | // Returns the maximum total_bytes() value for all existing |
| 409 | // DetachedBufferWriters. |
| 410 | // |
| 411 | // Returns 0 if no files are open. |
| 412 | size_t maximum_total_bytes() const { |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 413 | return accumulate_data_writers<size_t>( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 414 | 0, [](size_t x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 415 | CHECK_NOTNULL(data_writer.writer); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 416 | return std::max(x, data_writer.writer->total_bytes()); |
| 417 | }); |
Brian Silverman | 1f34522 | 2020-09-24 21:14:48 -0700 | [diff] [blame] | 418 | } |
| 419 | |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 420 | // Closes all existing log files. No more data may be written after this. |
| 421 | // |
| 422 | // This may set ran_out_of_space(). |
Austin Schuh | 08dba8f | 2023-05-01 08:29:30 -0700 | [diff] [blame] | 423 | WriteCode Close() override; |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 424 | |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 425 | // Accessors for various statistics. See the identically-named methods in |
| 426 | // DetachedBufferWriter for documentation. These are aggregated across all |
| 427 | // past and present DetachedBufferWriters. |
| 428 | std::chrono::nanoseconds max_write_time() const { |
| 429 | return accumulate_data_writers( |
| 430 | max_write_time_, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 431 | [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 432 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 433 | return std::max( |
| 434 | x, data_writer.writer->WriteStatistics()->max_write_time()); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 435 | }); |
| 436 | } |
| 437 | int max_write_time_bytes() const { |
| 438 | return std::get<0>(accumulate_data_writers( |
| 439 | std::make_tuple(max_write_time_bytes_, max_write_time_), |
| 440 | [](std::tuple<int, std::chrono::nanoseconds> x, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 441 | const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 442 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 443 | if (data_writer.writer->WriteStatistics()->max_write_time() > |
| 444 | std::get<1>(x)) { |
| 445 | return std::make_tuple( |
| 446 | data_writer.writer->WriteStatistics()->max_write_time_bytes(), |
| 447 | data_writer.writer->WriteStatistics()->max_write_time()); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 448 | } |
| 449 | return x; |
| 450 | })); |
| 451 | } |
| 452 | int max_write_time_messages() const { |
| 453 | return std::get<0>(accumulate_data_writers( |
| 454 | std::make_tuple(max_write_time_messages_, max_write_time_), |
| 455 | [](std::tuple<int, std::chrono::nanoseconds> x, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 456 | const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 457 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 458 | if (data_writer.writer->WriteStatistics()->max_write_time() > |
| 459 | std::get<1>(x)) { |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 460 | return std::make_tuple( |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 461 | data_writer.writer->WriteStatistics() |
| 462 | ->max_write_time_messages(), |
| 463 | data_writer.writer->WriteStatistics()->max_write_time()); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 464 | } |
| 465 | return x; |
| 466 | })); |
| 467 | } |
| 468 | std::chrono::nanoseconds total_write_time() const { |
| 469 | return accumulate_data_writers( |
| 470 | total_write_time_, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 471 | [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 472 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 473 | return x + data_writer.writer->WriteStatistics()->total_write_time(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 474 | }); |
| 475 | } |
| 476 | int total_write_count() const { |
| 477 | return accumulate_data_writers( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 478 | total_write_count_, [](int x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 479 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 480 | return x + data_writer.writer->WriteStatistics()->total_write_count(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 481 | }); |
| 482 | } |
| 483 | int total_write_messages() const { |
| 484 | return accumulate_data_writers( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 485 | total_write_messages_, [](int x, const NewDataWriter &data_writer) { |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 486 | return x + |
| 487 | data_writer.writer->WriteStatistics()->total_write_messages(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 488 | }); |
| 489 | } |
| 490 | int total_write_bytes() const { |
| 491 | return accumulate_data_writers( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 492 | total_write_bytes_, [](int x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 493 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 494 | return x + data_writer.writer->WriteStatistics()->total_write_bytes(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 495 | }); |
| 496 | } |
| 497 | |
| 498 | void ResetStatistics(); |
| 499 | |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 500 | protected: |
| 501 | // TODO (Alexei): consider to move ownership of log_namer to concrete sub |
| 502 | // class and make log_backend_ raw pointer. |
| 503 | LogBackend *log_backend() { return log_backend_.get(); } |
| 504 | const LogBackend *log_backend() const { return log_backend_.get(); } |
| 505 | |
Austin Schuh | 6ecfe90 | 2023-08-04 22:44:37 -0700 | [diff] [blame] | 506 | // Returns the data writer or timestamp writer if we find one for the provided |
| 507 | // node. |
| 508 | NewDataWriter *FindNodeDataWriter(const Node *node, size_t max_message_size); |
| 509 | NewDataWriter *FindNodeTimestampWriter(const Node *node, |
| 510 | size_t max_message_size); |
| 511 | |
| 512 | // Saves the data writer or timestamp writer for the provided node. |
| 513 | NewDataWriter *AddNodeDataWriter(const Node *node, NewDataWriter &&writer); |
| 514 | NewDataWriter *AddNodeTimestampWriter(const Node *node, |
| 515 | NewDataWriter &&writer); |
| 516 | |
| 517 | void CloseWriter(std::unique_ptr<DetachedBufferWriter> *writer_pointer); |
| 518 | |
| 519 | void CreateBufferWriter(std::string_view path, size_t max_message_size, |
| 520 | std::unique_ptr<DetachedBufferWriter> *destination); |
| 521 | |
| 522 | std::string extension_; |
| 523 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 524 | private: |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 525 | // Opens up a writer for timestamps forwarded back. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 526 | void OpenForwardedTimestampWriter(const Node *source_node, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 527 | NewDataWriter *data_writer); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 528 | |
| 529 | // Opens up a writer for remote data. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 530 | void OpenDataWriter(const Node *source_node, NewDataWriter *data_writer); |
| 531 | void OpenTimestampWriter(NewDataWriter *data_writer); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 532 | |
Austin Schuh | 6ecfe90 | 2023-08-04 22:44:37 -0700 | [diff] [blame] | 533 | // Tracks the node in nodes_. |
| 534 | void NoticeNode(const Node *source_node); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 535 | |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 536 | // A version of std::accumulate which operates over all of our DataWriters. |
| 537 | template <typename T, typename BinaryOperation> |
| 538 | T accumulate_data_writers(T t, BinaryOperation op) const { |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 539 | for (const std::pair<const Node *const, NewDataWriter> &data_writer : |
| 540 | node_data_writers_) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 541 | if (data_writer.second.writer != nullptr) { |
| 542 | t = op(std::move(t), data_writer.second); |
| 543 | } |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 544 | } |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 545 | for (const std::pair<const Node *const, NewDataWriter> &data_writer : |
| 546 | node_timestamp_writers_) { |
| 547 | if (data_writer.second.writer != nullptr) { |
| 548 | t = op(std::move(t), data_writer.second); |
| 549 | } |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 550 | } |
| 551 | return t; |
| 552 | } |
| 553 | |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 554 | std::unique_ptr<LogBackend> log_backend_; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 555 | |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 556 | bool ran_out_of_space_ = false; |
Brian Silverman | a621f52 | 2020-09-30 16:52:43 -0700 | [diff] [blame] | 557 | std::vector<std::string> all_filenames_; |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 558 | |
Austin Schuh | 8bdfc49 | 2023-02-11 12:53:13 -0800 | [diff] [blame] | 559 | std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory_; |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 560 | |
| 561 | // Storage for statistics from previously-rotated DetachedBufferWriters. |
| 562 | std::chrono::nanoseconds max_write_time_ = std::chrono::nanoseconds::zero(); |
| 563 | int max_write_time_bytes_ = -1; |
| 564 | int max_write_time_messages_ = -1; |
| 565 | std::chrono::nanoseconds total_write_time_ = std::chrono::nanoseconds::zero(); |
| 566 | int total_write_count_ = 0; |
| 567 | int total_write_messages_ = 0; |
| 568 | int total_write_bytes_ = 0; |
| 569 | |
Mithun Bharadwaj | 0c62993 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 570 | // Data writer per remote node. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 571 | std::map<const Node *, NewDataWriter> node_data_writers_; |
Mithun Bharadwaj | 99aec9e | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 572 | // Remote timestamp writers per node. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 573 | std::map<const Node *, NewDataWriter> node_timestamp_writers_; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 574 | }; |
| 575 | |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 576 | // This is specialized log namer that deals with directory centric log events. |
| 577 | class MultiNodeFilesLogNamer : public MultiNodeLogNamer { |
| 578 | public: |
| 579 | MultiNodeFilesLogNamer(std::string_view base_name, EventLoop *event_loop) |
colleen | 61276dc | 2023-06-01 09:23:29 -0700 | [diff] [blame] | 580 | : MultiNodeLogNamer( |
| 581 | std::make_unique<RenamableFileBackend>(base_name, false), |
| 582 | event_loop) {} |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 583 | |
| 584 | MultiNodeFilesLogNamer(std::string_view base_name, |
| 585 | const Configuration *configuration, |
| 586 | EventLoop *event_loop, const Node *node) |
colleen | 61276dc | 2023-06-01 09:23:29 -0700 | [diff] [blame] | 587 | : MultiNodeLogNamer( |
| 588 | std::make_unique<RenamableFileBackend>(base_name, false), |
| 589 | configuration, event_loop, node) {} |
| 590 | |
| 591 | MultiNodeFilesLogNamer(EventLoop *event_loop, |
| 592 | std::unique_ptr<RenamableFileBackend> backend) |
| 593 | : MultiNodeLogNamer(std::move(backend), event_loop) {} |
| 594 | |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 595 | ~MultiNodeFilesLogNamer() override = default; |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 596 | |
| 597 | std::string_view base_name() const { |
| 598 | return renamable_file_backend()->base_name(); |
| 599 | } |
| 600 | |
| 601 | // Rotate should be called at least once in between calls to set_base_name. |
| 602 | // Otherwise, temporary files will not be recoverable. |
| 603 | // Rotate is called by Logger::RenameLogBase, which is currently the only user |
| 604 | // of this method. |
| 605 | // Only renaming the folder is supported, not the file base name. |
| 606 | void set_base_name(std::string_view base_name) { |
| 607 | renamable_file_backend()->RenameLogBase(base_name); |
| 608 | } |
| 609 | |
| 610 | // When enabled, this will write files under names beginning |
| 611 | // with the .tmp suffix, and then rename them to the desired name after |
| 612 | // they are fully written. |
| 613 | // |
| 614 | // This is useful to enable incremental copying of the log files. |
| 615 | // |
| 616 | // Defaults to writing directly to the final filename. |
| 617 | void EnableTempFiles() { renamable_file_backend()->EnableTempFiles(); } |
| 618 | |
| 619 | private: |
| 620 | RenamableFileBackend *renamable_file_backend() { |
| 621 | return reinterpret_cast<RenamableFileBackend *>(log_backend()); |
| 622 | } |
| 623 | const RenamableFileBackend *renamable_file_backend() const { |
| 624 | return reinterpret_cast<const RenamableFileBackend *>(log_backend()); |
| 625 | } |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 626 | }; |
| 627 | |
Austin Schuh | 6ecfe90 | 2023-08-04 22:44:37 -0700 | [diff] [blame] | 628 | // Class which dumps all data from each node into a single file per node. This |
| 629 | // is mostly interesting for testing. |
| 630 | class MinimalFileMultiNodeLogNamer : public MultiNodeFilesLogNamer { |
| 631 | public: |
| 632 | MinimalFileMultiNodeLogNamer(std::string_view base_name, |
| 633 | EventLoop *event_loop) |
| 634 | : MultiNodeFilesLogNamer(base_name, event_loop) {} |
| 635 | MinimalFileMultiNodeLogNamer(std::string_view base_name, |
| 636 | const Configuration *configuration, |
| 637 | EventLoop *event_loop, const Node *node) |
| 638 | : MultiNodeFilesLogNamer(base_name, configuration, event_loop, node) {} |
| 639 | |
| 640 | NewDataWriter *MakeWriter(const Channel *channel) override; |
| 641 | |
| 642 | NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel, |
| 643 | const Node *node) override; |
| 644 | |
| 645 | NewDataWriter *MakeTimestampWriter(const Channel *channel) override; |
| 646 | |
| 647 | private: |
| 648 | // Names the data writer. |
| 649 | void OpenNodeWriter(const Node *source_node, NewDataWriter *data_writer); |
| 650 | }; |
| 651 | |
Stephan Pleines | d99b1ee | 2024-02-02 20:56:44 -0800 | [diff] [blame] | 652 | } // namespace aos::logger |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 653 | |
| 654 | #endif // AOS_EVENTS_LOGGING_LOG_NAMER_H_ |