Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 1 | #ifndef AOS_EVENTS_LOGGING_LOG_NAMER_H_ |
| 2 | #define AOS_EVENTS_LOGGING_LOG_NAMER_H_ |
| 3 | |
| 4 | #include <functional> |
| 5 | #include <map> |
| 6 | #include <memory> |
| 7 | #include <string_view> |
| 8 | #include <vector> |
| 9 | |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 10 | #include "absl/container/btree_map.h" |
Philipp Schrader | 790cb54 | 2023-07-05 21:06:52 -0700 | [diff] [blame] | 11 | #include "flatbuffers/flatbuffers.h" |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 12 | #include "glog/logging.h" |
Philipp Schrader | 790cb54 | 2023-07-05 21:06:52 -0700 | [diff] [blame] | 13 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 14 | #include "aos/events/logging/logfile_utils.h" |
| 15 | #include "aos/events/logging/logger_generated.h" |
Austin Schuh | 4385b14 | 2021-03-14 21:31:13 -0700 | [diff] [blame] | 16 | #include "aos/uuid.h" |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 17 | |
| 18 | namespace aos { |
| 19 | namespace logger { |
| 20 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 21 | class LogNamer; |
| 22 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 23 | // TODO(austin): Rename this back to DataWriter once all other callers are of |
| 24 | // the old DataWriter. |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 25 | // |
| 26 | // Class to manage writing data to log files. This lets us track which boot the |
| 27 | // written header has in it, and if the header has been written or not. |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 28 | // |
| 29 | // The design of this class is that instead of being notified when any of the |
| 30 | // header data changes, it polls and owns that decision. This makes it much |
| 31 | // harder to write corrupted data. If that becomes a performance problem, we |
| 32 | // can DCHECK and take it out of production binaries. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 33 | class NewDataWriter { |
| 34 | public: |
| 35 | // Constructs a NewDataWriter. |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 36 | // log_namer is the log namer which holds the config and any other data we |
| 37 | // need for our header. |
| 38 | // node is the node whom's prespective we are logging from. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 39 | // reopen is called whenever a file needs to be reopened. |
| 40 | // close is called to close that file and extract any statistics. |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 41 | NewDataWriter(LogNamer *log_namer, const Node *node, const Node *logger_node, |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 42 | std::function<void(NewDataWriter *)> reopen, |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 43 | std::function<void(NewDataWriter *)> close, |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 44 | size_t max_message_size, |
| 45 | std::initializer_list<StoredDataType> types); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 46 | |
| 47 | void UpdateMaxMessageSize(size_t new_size) { |
| 48 | if (new_size > max_message_size_) { |
Alexei Strots | bc082d8 | 2023-05-03 08:43:42 -0700 | [diff] [blame] | 49 | CHECK(!header_written_) << ": Tried to update to " << new_size << ", was " |
| 50 | << max_message_size_ << " for " << name(); |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 51 | max_message_size_ = new_size; |
| 52 | } |
| 53 | } |
| 54 | size_t max_message_size() const { return max_message_size_; } |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 55 | |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 56 | std::chrono::nanoseconds max_out_of_order_duration() const { |
| 57 | return max_out_of_order_duration_; |
| 58 | } |
| 59 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 60 | NewDataWriter(NewDataWriter &&other) = default; |
| 61 | aos::logger::NewDataWriter &operator=(NewDataWriter &&other) = default; |
| 62 | NewDataWriter(const NewDataWriter &) = delete; |
| 63 | void operator=(const NewDataWriter &) = delete; |
| 64 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 65 | ~NewDataWriter(); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 66 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 67 | // Rotates the log file, delaying writing the new header until data arrives. |
| 68 | void Rotate(); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 69 | |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 70 | // Updates all the metadata in the log file about the remote node which this |
| 71 | // message is from. |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 72 | void UpdateRemote(size_t remote_node_index, const UUID &remote_node_boot_uuid, |
| 73 | monotonic_clock::time_point monotonic_remote_time, |
| 74 | monotonic_clock::time_point monotonic_event_time, |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 75 | bool reliable, |
| 76 | monotonic_clock::time_point monotonic_timestamp_time = |
| 77 | monotonic_clock::min_time); |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 78 | |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 79 | // Coppies a message with the provided boot UUID. |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 80 | void CopyDataMessage(DataEncoder::Copier *copier, |
| 81 | const UUID &source_node_boot_uuid, |
| 82 | aos::monotonic_clock::time_point now, |
| 83 | aos::monotonic_clock::time_point message_time); |
| 84 | void CopyTimestampMessage(DataEncoder::Copier *copier, |
| 85 | const UUID &source_node_boot_uuid, |
| 86 | aos::monotonic_clock::time_point now, |
| 87 | aos::monotonic_clock::time_point message_time); |
| 88 | void CopyRemoteTimestampMessage( |
| 89 | DataEncoder::Copier *copier, const UUID &source_node_boot_uuid, |
| 90 | aos::monotonic_clock::time_point now, |
| 91 | aos::monotonic_clock::time_point message_time); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 92 | |
Austin Schuh | 5e14d84 | 2022-01-21 12:02:15 -0800 | [diff] [blame] | 93 | // Updates the current boot for the source node. This is useful when you want |
| 94 | // to queue a message that may trigger a reboot rotation, but then need to |
| 95 | // update the remote timestamps. |
| 96 | void UpdateBoot(const UUID &source_node_boot_uuid); |
| 97 | |
Alexei Strots | bc082d8 | 2023-05-03 08:43:42 -0700 | [diff] [blame] | 98 | // Returns the name of the writer. It may be a filename, but assume it is not. |
| 99 | std::string_view name() const { return writer ? writer->name() : "(closed)"; } |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 100 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 101 | void Close(); |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 102 | |
| 103 | std::unique_ptr<DetachedBufferWriter> writer = nullptr; |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 104 | |
| 105 | size_t node_index() const { return node_index_; } |
| 106 | const UUID &parts_uuid() const { return parts_uuid_; } |
| 107 | size_t parts_index() const { return parts_index_; } |
| 108 | const Node *node() const { return node_; } |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 109 | |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 110 | // Datastructure used to capture all the information about a remote node. |
| 111 | struct State { |
| 112 | // Boot UUID of the node. |
| 113 | UUID boot_uuid = UUID::Zero(); |
| 114 | // Timestamp on the remote monotonic clock of the oldest message sent to |
| 115 | // node_index_. |
| 116 | monotonic_clock::time_point oldest_remote_monotonic_timestamp = |
| 117 | monotonic_clock::max_time; |
| 118 | // Timestamp on the local monotonic clock of the message in |
| 119 | // oldest_remote_monotonic_timestamp. |
| 120 | monotonic_clock::time_point oldest_local_monotonic_timestamp = |
| 121 | monotonic_clock::max_time; |
| 122 | // Timestamp on the remote monotonic clock of the oldest message sent to |
| 123 | // node_index_, excluding messages forwarded with time_to_live() == 0. |
| 124 | monotonic_clock::time_point oldest_remote_unreliable_monotonic_timestamp = |
| 125 | monotonic_clock::max_time; |
| 126 | // Timestamp on the local monotonic clock of the message in |
| 127 | // oldest_local_unreliable_monotonic_timestamp. |
| 128 | monotonic_clock::time_point oldest_local_unreliable_monotonic_timestamp = |
| 129 | monotonic_clock::max_time; |
Austin Schuh | bfe6c57 | 2022-01-27 20:48:20 -0800 | [diff] [blame] | 130 | |
| 131 | // Timestamp on the remote monotonic clock of the oldest message sent to |
| 132 | // node_index_, only including messages forwarded with time_to_live() == 0. |
| 133 | monotonic_clock::time_point oldest_remote_reliable_monotonic_timestamp = |
| 134 | monotonic_clock::max_time; |
| 135 | // Timestamp on the local monotonic clock of the message in |
| 136 | // oldest_local_reliable_monotonic_timestamp. |
| 137 | monotonic_clock::time_point oldest_local_reliable_monotonic_timestamp = |
| 138 | monotonic_clock::max_time; |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 139 | |
| 140 | // Timestamp on the remote monotonic clock of the oldest message timestamp |
| 141 | // sent back to logger_node_index_. The remote here will be the node this |
| 142 | // part is from the perspective of, ie node_index_. |
| 143 | monotonic_clock::time_point |
| 144 | oldest_logger_remote_unreliable_monotonic_timestamp = |
| 145 | monotonic_clock::max_time; |
| 146 | // The time on the monotonic clock of the logger when this timestamp made it |
| 147 | // back to the logger (logger_node_index_). |
| 148 | monotonic_clock::time_point |
| 149 | oldest_logger_local_unreliable_monotonic_timestamp = |
| 150 | monotonic_clock::max_time; |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 151 | }; |
| 152 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 153 | private: |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 154 | // Signals that a node has rebooted. |
Austin Schuh | 5e14d84 | 2022-01-21 12:02:15 -0800 | [diff] [blame] | 155 | void Reboot(const UUID &source_node_boot_uuid); |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 156 | |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 157 | void CopyMessage(DataEncoder::Copier *copier, |
| 158 | const UUID &source_node_boot_uuid, |
| 159 | aos::monotonic_clock::time_point now, |
| 160 | aos::monotonic_clock::time_point message_time); |
| 161 | |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 162 | void QueueHeader( |
| 163 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> &&header); |
| 164 | |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 165 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader(); |
| 166 | |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 167 | monotonic_clock::time_point monotonic_start_time_ = monotonic_clock::min_time; |
| 168 | |
Austin Schuh | 577610e | 2021-12-08 12:07:19 -0800 | [diff] [blame] | 169 | const Node *node_ = nullptr; |
| 170 | size_t node_index_ = 0; |
Austin Schuh | f5f99f3 | 2022-02-07 20:05:37 -0800 | [diff] [blame] | 171 | size_t logger_node_index_ = 0; |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 172 | LogNamer *log_namer_; |
| 173 | UUID parts_uuid_ = UUID::Random(); |
| 174 | size_t parts_index_ = 0; |
| 175 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 176 | std::function<void(NewDataWriter *)> reopen_; |
| 177 | std::function<void(NewDataWriter *)> close_; |
Austin Schuh | 572924a | 2021-07-30 22:32:12 -0700 | [diff] [blame] | 178 | bool header_written_ = false; |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 179 | |
Austin Schuh | 72211ae | 2021-08-05 14:02:30 -0700 | [diff] [blame] | 180 | std::vector<State> state_; |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 181 | |
| 182 | size_t max_message_size_; |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 183 | |
| 184 | // Each data writer logs the channels for that node, i.e. |
| 185 | // each data writer writes one file. We may encounter messages which |
| 186 | // violate the max out of order duration specified in the header of that file. |
| 187 | // Rotate the data writer and start a new part for that particular file. |
| 188 | // This shouldn't affect the headers of other data writers, so make this |
| 189 | // a property of individual data writer instead of the overall log. |
| 190 | std::chrono::nanoseconds max_out_of_order_duration_; |
| 191 | |
| 192 | // Monotonic time point of the latest message we've logged so far, i.e |
| 193 | // Message X - time Z |
| 194 | // Message Y - time Z + 1 |
| 195 | // newest_message_time_ = Z + 1 (even if X was logged after Y) |
| 196 | // |
| 197 | // Since the messages can be logged out of order, this helps determine if |
| 198 | // max out of order duration was violated. |
| 199 | monotonic_clock::time_point newest_message_time_ = monotonic_clock::min_time; |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 200 | |
| 201 | // An array with a bool for each value of StoredDataType representing if that |
| 202 | // data type is allowed to be logged by this object. |
| 203 | std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1> |
| 204 | allowed_data_types_; |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 205 | }; |
| 206 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 207 | // Interface describing how to name, track, and add headers to log file parts. |
| 208 | class LogNamer { |
| 209 | public: |
| 210 | // Constructs a LogNamer with the primary node (ie the one the logger runs on) |
| 211 | // being node. |
Austin Schuh | 5b728b7 | 2021-06-16 14:57:15 -0700 | [diff] [blame] | 212 | LogNamer(const aos::Configuration *configuration, EventLoop *event_loop, |
| 213 | const aos::Node *node) |
Austin Schuh | a499cea | 2021-07-31 19:49:53 -0700 | [diff] [blame] | 214 | : event_loop_(event_loop), |
Austin Schuh | 5b728b7 | 2021-06-16 14:57:15 -0700 | [diff] [blame] | 215 | configuration_(configuration), |
| 216 | node_(node), |
Austin Schuh | a499cea | 2021-07-31 19:49:53 -0700 | [diff] [blame] | 217 | logger_node_index_(configuration::GetNodeIndex(configuration_, node_)) { |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 218 | nodes_.emplace_back(node_); |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 219 | } |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 220 | virtual ~LogNamer() = default; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 221 | |
Brian Silverman | 87ac040 | 2020-09-17 14:47:01 -0700 | [diff] [blame] | 222 | // Returns a writer for writing data from messages on this channel (on the |
| 223 | // primary node). |
| 224 | // |
| 225 | // The returned pointer will stay valid across rotations, but the object it |
| 226 | // points to will be assigned to. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 227 | virtual NewDataWriter *MakeWriter(const Channel *channel) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 228 | |
Brian Silverman | 87ac040 | 2020-09-17 14:47:01 -0700 | [diff] [blame] | 229 | // Returns a writer for writing timestamps from messages on this channel (on |
| 230 | // the primary node). |
| 231 | // |
| 232 | // The returned pointer will stay valid across rotations, but the object it |
| 233 | // points to will be assigned to. |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 234 | virtual NewDataWriter *MakeTimestampWriter(const Channel *channel) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 235 | |
| 236 | // Returns a writer for writing timestamps delivered over the special |
| 237 | // /aos/remote_timestamps/* channels. node is the node that the timestamps |
Brian Silverman | 87ac040 | 2020-09-17 14:47:01 -0700 | [diff] [blame] | 238 | // are forwarded back from (to the primary node). |
| 239 | // |
| 240 | // The returned pointer will stay valid across rotations, but the object it |
| 241 | // points to will be assigned to. |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 242 | virtual NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel, |
| 243 | const Node *node) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 244 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 245 | // Rotates all log files for the provided node. |
| 246 | virtual void Rotate(const Node *node) = 0; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 247 | |
| 248 | // Returns all the nodes that data is being written for. |
| 249 | const std::vector<const Node *> &nodes() const { return nodes_; } |
| 250 | |
Austin Schuh | 08dba8f | 2023-05-01 08:29:30 -0700 | [diff] [blame] | 251 | // Closes all existing log data writers. No more data may be written after |
| 252 | // this. |
| 253 | virtual WriteCode Close() = 0; |
| 254 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 255 | // Returns the node the logger is running on. |
| 256 | const Node *node() const { return node_; } |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 257 | const UUID &logger_node_boot_uuid() const { return logger_node_boot_uuid_; } |
| 258 | size_t logger_node_index() const { return logger_node_index_; } |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 259 | |
Austin Schuh | 8c39996 | 2020-12-25 21:51:45 -0800 | [diff] [blame] | 260 | // Writes out the nested Configuration object to the config file location. |
| 261 | virtual void WriteConfiguration( |
| 262 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header, |
| 263 | std::string_view config_sha256) = 0; |
| 264 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 265 | void SetHeaderTemplate( |
| 266 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header) { |
| 267 | header_ = std::move(header); |
Austin Schuh | e46492f | 2021-07-31 19:49:41 -0700 | [diff] [blame] | 268 | logger_node_boot_uuid_ = |
| 269 | UUID::FromString(header_.message().logger_node_boot_uuid()); |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 270 | } |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 271 | |
Austin Schuh | 60e7794 | 2022-05-16 17:48:24 -0700 | [diff] [blame] | 272 | void ClearStartTimes() { node_states_.clear(); } |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 273 | |
| 274 | void SetStartTimes(size_t node_index, const UUID &boot_uuid, |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 275 | monotonic_clock::time_point monotonic_start_time, |
| 276 | realtime_clock::time_point realtime_start_time, |
| 277 | monotonic_clock::time_point logger_monotonic_start_time, |
| 278 | realtime_clock::time_point logger_realtime_start_time) { |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 279 | VLOG(1) << "Setting node " << node_index << " to start time " |
| 280 | << monotonic_start_time << " rt " << realtime_start_time << " UUID " |
| 281 | << boot_uuid; |
| 282 | NodeState *node_state = GetNodeState(node_index, boot_uuid); |
| 283 | node_state->monotonic_start_time = monotonic_start_time; |
| 284 | node_state->realtime_start_time = realtime_start_time; |
| 285 | node_state->logger_monotonic_start_time = logger_monotonic_start_time; |
| 286 | node_state->logger_realtime_start_time = logger_realtime_start_time; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 287 | } |
| 288 | |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 289 | monotonic_clock::time_point monotonic_start_time(size_t node_index, |
| 290 | const UUID &boot_uuid) { |
| 291 | DCHECK_NE(boot_uuid, UUID::Zero()); |
| 292 | |
| 293 | NodeState *node_state = GetNodeState(node_index, boot_uuid); |
| 294 | return node_state->monotonic_start_time; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 295 | } |
| 296 | |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 297 | // This returns the initial out of order duration set in the header template |
| 298 | // by the logger based on polling period. It may be different than the actual |
| 299 | // duration used by the data writer. |
| 300 | std::chrono::nanoseconds base_max_out_of_order_duration() const { |
| 301 | return std::chrono::nanoseconds( |
| 302 | header_.message().max_out_of_order_duration()); |
| 303 | } |
| 304 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 305 | protected: |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 306 | // Structure with state per node about times and such. |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 307 | struct NodeState { |
| 308 | // Time when this node started logging. |
| 309 | monotonic_clock::time_point monotonic_start_time = |
| 310 | monotonic_clock::min_time; |
| 311 | realtime_clock::time_point realtime_start_time = realtime_clock::min_time; |
| 312 | |
| 313 | // Corresponding time on the logger node when it started logging. |
| 314 | monotonic_clock::time_point logger_monotonic_start_time = |
| 315 | monotonic_clock::min_time; |
| 316 | realtime_clock::time_point logger_realtime_start_time = |
| 317 | realtime_clock::min_time; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 318 | }; |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 319 | |
| 320 | // Creates a new header by copying fields out of the template and combining |
| 321 | // them with the arguments provided. |
| 322 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> MakeHeader( |
| 323 | size_t node_index, const std::vector<NewDataWriter::State> &state, |
Mithun Bharadwaj | a5cb8e0 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 324 | const UUID &parts_uuid, int parts_index, |
Mithun Bharadwaj | a5f9d48 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 325 | std::chrono::nanoseconds max_out_of_order_duration, |
| 326 | const std::array<bool, static_cast<size_t>(StoredDataType::MAX) + 1> |
| 327 | &allowed_data_types); |
Austin Schuh | 58646e2 | 2021-08-23 23:51:46 -0700 | [diff] [blame] | 328 | |
| 329 | EventLoop *event_loop_; |
| 330 | const Configuration *const configuration_; |
| 331 | const Node *const node_; |
| 332 | const size_t logger_node_index_; |
| 333 | UUID logger_node_boot_uuid_; |
| 334 | std::vector<const Node *> nodes_; |
| 335 | |
| 336 | friend NewDataWriter; |
| 337 | |
| 338 | // Returns the start/stop time state structure for a node and boot. We can |
| 339 | // have data from multiple boots, and it makes sense to reuse the start/stop |
| 340 | // times if we get data from the same boot again. |
| 341 | NodeState *GetNodeState(size_t node_index, const UUID &boot_uuid); |
| 342 | |
| 343 | absl::btree_map<std::pair<size_t, UUID>, NodeState> node_states_; |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 344 | |
| 345 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> header_ = |
| 346 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader>::Empty(); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 347 | }; |
| 348 | |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 349 | // Log namer which uses a config to name a bunch of files. |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 350 | class MultiNodeLogNamer : public LogNamer { |
| 351 | public: |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 352 | MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend, |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 353 | EventLoop *event_loop); |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 354 | MultiNodeLogNamer(std::unique_ptr<LogBackend> log_backend, |
Austin Schuh | 5b728b7 | 2021-06-16 14:57:15 -0700 | [diff] [blame] | 355 | const Configuration *configuration, EventLoop *event_loop, |
| 356 | const Node *node); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 357 | ~MultiNodeLogNamer() override; |
| 358 | |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 359 | // Sets the function for creating encoders. The argument is the max message |
| 360 | // size (including headers) that will be written into this encoder. |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 361 | // |
| 362 | // Defaults to just creating DummyEncoders. |
| 363 | void set_encoder_factory( |
Austin Schuh | 48d10d6 | 2022-10-16 22:19:23 -0700 | [diff] [blame] | 364 | std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory) { |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 365 | encoder_factory_ = std::move(encoder_factory); |
| 366 | } |
| 367 | |
| 368 | // Sets an additional file extension. |
| 369 | // |
| 370 | // Defaults to nothing. |
| 371 | void set_extension(std::string_view extension) { extension_ = extension; } |
Brian Silverman | 1f34522 | 2020-09-24 21:14:48 -0700 | [diff] [blame] | 372 | |
Brian Silverman | a621f52 | 2020-09-30 16:52:43 -0700 | [diff] [blame] | 373 | // A list of all the filenames we've written. |
| 374 | // |
| 375 | // This only includes the part after base_name(). |
| 376 | const std::vector<std::string> &all_filenames() const { |
| 377 | return all_filenames_; |
| 378 | } |
| 379 | |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 380 | void Rotate(const Node *node) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 381 | |
Austin Schuh | 8c39996 | 2020-12-25 21:51:45 -0800 | [diff] [blame] | 382 | void WriteConfiguration( |
| 383 | aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header, |
| 384 | std::string_view config_sha256) override; |
| 385 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 386 | NewDataWriter *MakeWriter(const Channel *channel) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 387 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 388 | NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel, |
Austin Schuh | 7334084 | 2021-07-30 22:32:06 -0700 | [diff] [blame] | 389 | const Node *node) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 390 | |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 391 | NewDataWriter *MakeTimestampWriter(const Channel *channel) override; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 392 | |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 393 | // Indicates that at least one file ran out of space. Once this happens, we |
| 394 | // stop trying to open new files, to avoid writing any files with holes from |
| 395 | // previous parts. |
| 396 | // |
| 397 | // Besides this function, this object will silently stop logging data when |
| 398 | // this occurs. If you want to ensure log files are complete, you must call |
| 399 | // this method. |
Brian Silverman | a9f2ec9 | 2020-10-06 18:00:53 -0700 | [diff] [blame] | 400 | bool ran_out_of_space() const { |
| 401 | return accumulate_data_writers<bool>( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 402 | ran_out_of_space_, [](bool x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 403 | CHECK_NOTNULL(data_writer.writer); |
Brian Silverman | a9f2ec9 | 2020-10-06 18:00:53 -0700 | [diff] [blame] | 404 | return x || |
| 405 | (data_writer.writer && data_writer.writer->ran_out_of_space()); |
| 406 | }); |
| 407 | } |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 408 | |
Brian Silverman | 1f34522 | 2020-09-24 21:14:48 -0700 | [diff] [blame] | 409 | // Returns the maximum total_bytes() value for all existing |
| 410 | // DetachedBufferWriters. |
| 411 | // |
| 412 | // Returns 0 if no files are open. |
| 413 | size_t maximum_total_bytes() const { |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 414 | return accumulate_data_writers<size_t>( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 415 | 0, [](size_t x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 416 | CHECK_NOTNULL(data_writer.writer); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 417 | return std::max(x, data_writer.writer->total_bytes()); |
| 418 | }); |
Brian Silverman | 1f34522 | 2020-09-24 21:14:48 -0700 | [diff] [blame] | 419 | } |
| 420 | |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 421 | // Closes all existing log files. No more data may be written after this. |
| 422 | // |
| 423 | // This may set ran_out_of_space(). |
Austin Schuh | 08dba8f | 2023-05-01 08:29:30 -0700 | [diff] [blame] | 424 | WriteCode Close() override; |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 425 | |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 426 | // Accessors for various statistics. See the identically-named methods in |
| 427 | // DetachedBufferWriter for documentation. These are aggregated across all |
| 428 | // past and present DetachedBufferWriters. |
| 429 | std::chrono::nanoseconds max_write_time() const { |
| 430 | return accumulate_data_writers( |
| 431 | max_write_time_, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 432 | [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 433 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 434 | return std::max( |
| 435 | x, data_writer.writer->WriteStatistics()->max_write_time()); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 436 | }); |
| 437 | } |
| 438 | int max_write_time_bytes() const { |
| 439 | return std::get<0>(accumulate_data_writers( |
| 440 | std::make_tuple(max_write_time_bytes_, max_write_time_), |
| 441 | [](std::tuple<int, std::chrono::nanoseconds> x, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 442 | const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 443 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 444 | if (data_writer.writer->WriteStatistics()->max_write_time() > |
| 445 | std::get<1>(x)) { |
| 446 | return std::make_tuple( |
| 447 | data_writer.writer->WriteStatistics()->max_write_time_bytes(), |
| 448 | data_writer.writer->WriteStatistics()->max_write_time()); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 449 | } |
| 450 | return x; |
| 451 | })); |
| 452 | } |
| 453 | int max_write_time_messages() const { |
| 454 | return std::get<0>(accumulate_data_writers( |
| 455 | std::make_tuple(max_write_time_messages_, max_write_time_), |
| 456 | [](std::tuple<int, std::chrono::nanoseconds> x, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 457 | const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 458 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 459 | if (data_writer.writer->WriteStatistics()->max_write_time() > |
| 460 | std::get<1>(x)) { |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 461 | return std::make_tuple( |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 462 | data_writer.writer->WriteStatistics() |
| 463 | ->max_write_time_messages(), |
| 464 | data_writer.writer->WriteStatistics()->max_write_time()); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 465 | } |
| 466 | return x; |
| 467 | })); |
| 468 | } |
| 469 | std::chrono::nanoseconds total_write_time() const { |
| 470 | return accumulate_data_writers( |
| 471 | total_write_time_, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 472 | [](std::chrono::nanoseconds x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 473 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 474 | return x + data_writer.writer->WriteStatistics()->total_write_time(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 475 | }); |
| 476 | } |
| 477 | int total_write_count() const { |
| 478 | return accumulate_data_writers( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 479 | total_write_count_, [](int x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 480 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 481 | return x + data_writer.writer->WriteStatistics()->total_write_count(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 482 | }); |
| 483 | } |
| 484 | int total_write_messages() const { |
| 485 | return accumulate_data_writers( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 486 | total_write_messages_, [](int x, const NewDataWriter &data_writer) { |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 487 | return x + |
| 488 | data_writer.writer->WriteStatistics()->total_write_messages(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 489 | }); |
| 490 | } |
| 491 | int total_write_bytes() const { |
| 492 | return accumulate_data_writers( |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 493 | total_write_bytes_, [](int x, const NewDataWriter &data_writer) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 494 | CHECK_NOTNULL(data_writer.writer); |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 495 | return x + data_writer.writer->WriteStatistics()->total_write_bytes(); |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 496 | }); |
| 497 | } |
| 498 | |
| 499 | void ResetStatistics(); |
| 500 | |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 501 | protected: |
| 502 | // TODO (Alexei): consider to move ownership of log_namer to concrete sub |
| 503 | // class and make log_backend_ raw pointer. |
| 504 | LogBackend *log_backend() { return log_backend_.get(); } |
| 505 | const LogBackend *log_backend() const { return log_backend_.get(); } |
| 506 | |
Austin Schuh | 6ecfe90 | 2023-08-04 22:44:37 -0700 | [diff] [blame^] | 507 | // Returns the data writer or timestamp writer if we find one for the provided |
| 508 | // node. |
| 509 | NewDataWriter *FindNodeDataWriter(const Node *node, size_t max_message_size); |
| 510 | NewDataWriter *FindNodeTimestampWriter(const Node *node, |
| 511 | size_t max_message_size); |
| 512 | |
| 513 | // Saves the data writer or timestamp writer for the provided node. |
| 514 | NewDataWriter *AddNodeDataWriter(const Node *node, NewDataWriter &&writer); |
| 515 | NewDataWriter *AddNodeTimestampWriter(const Node *node, |
| 516 | NewDataWriter &&writer); |
| 517 | |
| 518 | void CloseWriter(std::unique_ptr<DetachedBufferWriter> *writer_pointer); |
| 519 | |
| 520 | void CreateBufferWriter(std::string_view path, size_t max_message_size, |
| 521 | std::unique_ptr<DetachedBufferWriter> *destination); |
| 522 | |
| 523 | std::string extension_; |
| 524 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 525 | private: |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 526 | // Opens up a writer for timestamps forwarded back. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 527 | void OpenForwardedTimestampWriter(const Node *source_node, |
Austin Schuh | b8bca73 | 2021-07-30 22:32:00 -0700 | [diff] [blame] | 528 | NewDataWriter *data_writer); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 529 | |
| 530 | // Opens up a writer for remote data. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 531 | void OpenDataWriter(const Node *source_node, NewDataWriter *data_writer); |
| 532 | void OpenTimestampWriter(NewDataWriter *data_writer); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 533 | |
Austin Schuh | 6ecfe90 | 2023-08-04 22:44:37 -0700 | [diff] [blame^] | 534 | // Tracks the node in nodes_. |
| 535 | void NoticeNode(const Node *source_node); |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 536 | |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 537 | // A version of std::accumulate which operates over all of our DataWriters. |
| 538 | template <typename T, typename BinaryOperation> |
| 539 | T accumulate_data_writers(T t, BinaryOperation op) const { |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 540 | for (const std::pair<const Node *const, NewDataWriter> &data_writer : |
| 541 | node_data_writers_) { |
Maxwell Gumley | 8ad7778 | 2023-07-11 13:27:03 -0600 | [diff] [blame] | 542 | if (data_writer.second.writer != nullptr) { |
| 543 | t = op(std::move(t), data_writer.second); |
| 544 | } |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 545 | } |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 546 | for (const std::pair<const Node *const, NewDataWriter> &data_writer : |
| 547 | node_timestamp_writers_) { |
| 548 | if (data_writer.second.writer != nullptr) { |
| 549 | t = op(std::move(t), data_writer.second); |
| 550 | } |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 551 | } |
| 552 | return t; |
| 553 | } |
| 554 | |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 555 | std::unique_ptr<LogBackend> log_backend_; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 556 | |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 557 | bool ran_out_of_space_ = false; |
Brian Silverman | a621f52 | 2020-09-30 16:52:43 -0700 | [diff] [blame] | 558 | std::vector<std::string> all_filenames_; |
Brian Silverman | 0465fcf | 2020-09-24 00:29:18 -0700 | [diff] [blame] | 559 | |
Austin Schuh | 8bdfc49 | 2023-02-11 12:53:13 -0800 | [diff] [blame] | 560 | std::function<std::unique_ptr<DataEncoder>(size_t)> encoder_factory_; |
Brian Silverman | cb80582 | 2020-10-06 17:43:35 -0700 | [diff] [blame] | 561 | |
| 562 | // Storage for statistics from previously-rotated DetachedBufferWriters. |
| 563 | std::chrono::nanoseconds max_write_time_ = std::chrono::nanoseconds::zero(); |
| 564 | int max_write_time_bytes_ = -1; |
| 565 | int max_write_time_messages_ = -1; |
| 566 | std::chrono::nanoseconds total_write_time_ = std::chrono::nanoseconds::zero(); |
| 567 | int total_write_count_ = 0; |
| 568 | int total_write_messages_ = 0; |
| 569 | int total_write_bytes_ = 0; |
| 570 | |
Mithun Bharadwaj | 0c62993 | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 571 | // Data writer per remote node. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 572 | std::map<const Node *, NewDataWriter> node_data_writers_; |
Mithun Bharadwaj | 99aec9e | 2023-08-02 16:10:40 -0700 | [diff] [blame] | 573 | // Remote timestamp writers per node. |
Mithun Bharadwaj | c54aa02 | 2023-08-02 16:10:41 -0700 | [diff] [blame] | 574 | std::map<const Node *, NewDataWriter> node_timestamp_writers_; |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 575 | }; |
| 576 | |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 577 | // This is specialized log namer that deals with directory centric log events. |
| 578 | class MultiNodeFilesLogNamer : public MultiNodeLogNamer { |
| 579 | public: |
| 580 | MultiNodeFilesLogNamer(std::string_view base_name, EventLoop *event_loop) |
colleen | 61276dc | 2023-06-01 09:23:29 -0700 | [diff] [blame] | 581 | : MultiNodeLogNamer( |
| 582 | std::make_unique<RenamableFileBackend>(base_name, false), |
| 583 | event_loop) {} |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 584 | |
| 585 | MultiNodeFilesLogNamer(std::string_view base_name, |
| 586 | const Configuration *configuration, |
| 587 | EventLoop *event_loop, const Node *node) |
colleen | 61276dc | 2023-06-01 09:23:29 -0700 | [diff] [blame] | 588 | : MultiNodeLogNamer( |
| 589 | std::make_unique<RenamableFileBackend>(base_name, false), |
| 590 | configuration, event_loop, node) {} |
| 591 | |
| 592 | MultiNodeFilesLogNamer(EventLoop *event_loop, |
| 593 | std::unique_ptr<RenamableFileBackend> backend) |
| 594 | : MultiNodeLogNamer(std::move(backend), event_loop) {} |
| 595 | |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 596 | ~MultiNodeFilesLogNamer() override = default; |
Alexei Strots | caf17d3 | 2023-04-03 22:31:11 -0700 | [diff] [blame] | 597 | |
| 598 | std::string_view base_name() const { |
| 599 | return renamable_file_backend()->base_name(); |
| 600 | } |
| 601 | |
| 602 | // Rotate should be called at least once in between calls to set_base_name. |
| 603 | // Otherwise, temporary files will not be recoverable. |
| 604 | // Rotate is called by Logger::RenameLogBase, which is currently the only user |
| 605 | // of this method. |
| 606 | // Only renaming the folder is supported, not the file base name. |
| 607 | void set_base_name(std::string_view base_name) { |
| 608 | renamable_file_backend()->RenameLogBase(base_name); |
| 609 | } |
| 610 | |
| 611 | // When enabled, this will write files under names beginning |
| 612 | // with the .tmp suffix, and then rename them to the desired name after |
| 613 | // they are fully written. |
| 614 | // |
| 615 | // This is useful to enable incremental copying of the log files. |
| 616 | // |
| 617 | // Defaults to writing directly to the final filename. |
| 618 | void EnableTempFiles() { renamable_file_backend()->EnableTempFiles(); } |
| 619 | |
| 620 | private: |
| 621 | RenamableFileBackend *renamable_file_backend() { |
| 622 | return reinterpret_cast<RenamableFileBackend *>(log_backend()); |
| 623 | } |
| 624 | const RenamableFileBackend *renamable_file_backend() const { |
| 625 | return reinterpret_cast<const RenamableFileBackend *>(log_backend()); |
| 626 | } |
Alexei Strots | 0139549 | 2023-03-20 13:59:56 -0700 | [diff] [blame] | 627 | }; |
| 628 | |
Austin Schuh | 6ecfe90 | 2023-08-04 22:44:37 -0700 | [diff] [blame^] | 629 | // Class which dumps all data from each node into a single file per node. This |
| 630 | // is mostly interesting for testing. |
| 631 | class MinimalFileMultiNodeLogNamer : public MultiNodeFilesLogNamer { |
| 632 | public: |
| 633 | MinimalFileMultiNodeLogNamer(std::string_view base_name, |
| 634 | EventLoop *event_loop) |
| 635 | : MultiNodeFilesLogNamer(base_name, event_loop) {} |
| 636 | MinimalFileMultiNodeLogNamer(std::string_view base_name, |
| 637 | const Configuration *configuration, |
| 638 | EventLoop *event_loop, const Node *node) |
| 639 | : MultiNodeFilesLogNamer(base_name, configuration, event_loop, node) {} |
| 640 | |
| 641 | NewDataWriter *MakeWriter(const Channel *channel) override; |
| 642 | |
| 643 | NewDataWriter *MakeForwardedTimestampWriter(const Channel *channel, |
| 644 | const Node *node) override; |
| 645 | |
| 646 | NewDataWriter *MakeTimestampWriter(const Channel *channel) override; |
| 647 | |
| 648 | private: |
| 649 | // Names the data writer. |
| 650 | void OpenNodeWriter(const Node *source_node, NewDataWriter *data_writer); |
| 651 | }; |
| 652 | |
Austin Schuh | cb5601b | 2020-09-10 15:29:59 -0700 | [diff] [blame] | 653 | } // namespace logger |
| 654 | } // namespace aos |
| 655 | |
| 656 | #endif // AOS_EVENTS_LOGGING_LOG_NAMER_H_ |