Support graceful logging failures when running out of disk space
It's the most obvious failure mode when logging, so let's allow the
higher-level code to report a status instead of just crash-looping.
Change-Id: Iff223fd8b6b0f7f4b21d154a4dda5cce80fa6af2
diff --git a/aos/events/logging/log_namer.cc b/aos/events/logging/log_namer.cc
index a635682..a332929 100644
--- a/aos/events/logging/log_namer.cc
+++ b/aos/events/logging/log_namer.cc
@@ -188,11 +188,11 @@
DetachedBufferWriter *MultiNodeLogNamer::MakeTimestampWriter(
const Channel *channel) {
- const bool log_delivery_times =
- (this->node() == nullptr)
- ? false
- : configuration::ConnectionDeliveryTimeIsLoggedOnNode(
- channel, this->node(), this->node());
+ bool log_delivery_times = false;
+ if (this->node() != nullptr) {
+ log_delivery_times = configuration::ConnectionDeliveryTimeIsLoggedOnNode(
+ channel, this->node(), this->node());
+ }
if (!log_delivery_times) {
return nullptr;
}
@@ -200,20 +200,33 @@
return data_writer_.get();
}
+void MultiNodeLogNamer::Close() {
+ for (std::pair<const Channel *const, DataWriter> &data_writer :
+ data_writers_) {
+ if (data_writer.second.writer) {
+ data_writer.second.writer->Close();
+ if (data_writer.second.writer->ran_out_of_space()) {
+ ran_out_of_space_ = true;
+ data_writer.second.writer->acknowledge_out_of_space();
+ }
+ }
+ }
+ if (data_writer_) {
+ data_writer_->Close();
+ if (data_writer_->ran_out_of_space()) {
+ ran_out_of_space_ = true;
+ data_writer_->acknowledge_out_of_space();
+ }
+ }
+}
+
void MultiNodeLogNamer::OpenForwardedTimestampWriter(const Channel *channel,
DataWriter *data_writer) {
std::string filename =
absl::StrCat(base_name_, "_timestamps", channel->name()->string_view(),
"/", channel->type()->string_view(), ".part",
data_writer->part_number, ".bfbs");
-
- if (!data_writer->writer) {
- data_writer->writer = std::make_unique<DetachedBufferWriter>(
- filename, std::make_unique<DummyEncoder>());
- } else {
- *data_writer->writer =
- DetachedBufferWriter(filename, std::make_unique<DummyEncoder>());
- }
+ CreateBufferWriter(filename, &data_writer->writer);
}
void MultiNodeLogNamer::OpenWriter(const Channel *channel,
@@ -222,13 +235,7 @@
base_name_, "_", channel->source_node()->string_view(), "_data",
channel->name()->string_view(), "/", channel->type()->string_view(),
".part", data_writer->part_number, ".bfbs");
- if (!data_writer->writer) {
- data_writer->writer = std::make_unique<DetachedBufferWriter>(
- filename, std::make_unique<DummyEncoder>());
- } else {
- *data_writer->writer =
- DetachedBufferWriter(filename, std::make_unique<DummyEncoder>());
- }
+ CreateBufferWriter(filename, &data_writer->writer);
}
std::unique_ptr<DetachedBufferWriter> MultiNodeLogNamer::OpenDataWriter() {
@@ -238,5 +245,28 @@
std::make_unique<DummyEncoder>());
}
+void MultiNodeLogNamer::CreateBufferWriter(
+ std::string_view filename,
+ std::unique_ptr<DetachedBufferWriter> *destination) {
+ if (ran_out_of_space_) {
+ // Refuse to open any new files, which might skip data. Any existing files
+ // are in the same folder, which means they're on the same filesystem, which
+ // means they're probably going to run out of space and get stuck too.
+ return;
+ }
+ if (!destination->get()) {
+ *destination = std::make_unique<DetachedBufferWriter>(
+ filename, std::make_unique<DummyEncoder>());
+ return;
+ }
+ destination->get()->Close();
+ if (destination->get()->ran_out_of_space()) {
+ ran_out_of_space_ = true;
+ return;
+ }
+ *destination->get() =
+ DetachedBufferWriter(filename, std::make_unique<DummyEncoder>());
+}
+
} // namespace logger
} // namespace aos