Merge changes I8bdc9afc,If85e515c
* changes:
Teach MultiNodeLogNamer to write files with a temporary suffix
Expose a list of all the part files which were created
diff --git a/aos/events/logging/log_namer.cc b/aos/events/logging/log_namer.cc
index 8dd5061..06243b0 100644
--- a/aos/events/logging/log_namer.cc
+++ b/aos/events/logging/log_namer.cc
@@ -69,12 +69,22 @@
MultiNodeLogNamer::MultiNodeLogNamer(std::string_view base_name,
const Configuration *configuration,
- const Node *node)
+ const Node *node,
+ std::string_view temp_suffix)
: LogNamer(node),
base_name_(base_name),
+ temp_suffix_(temp_suffix),
configuration_(configuration),
- uuid_(UUID::Random()),
- data_writer_(OpenDataWriter()) {}
+ uuid_(UUID::Random()) {
+ OpenDataWriter();
+}
+
+MultiNodeLogNamer::~MultiNodeLogNamer() {
+ if (!ran_out_of_space_) {
+ // This handles renaming temporary files etc.
+ Close();
+ }
+}
void MultiNodeLogNamer::WriteHeader(
aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header,
@@ -99,7 +109,7 @@
aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header) {
if (node == this->node()) {
++part_number_;
- *data_writer_ = std::move(*OpenDataWriter());
+ OpenDataWriter();
UpdateHeader(header, uuid_, part_number_);
data_writer_->QueueSpan(header->full_span());
} else {
@@ -209,6 +219,8 @@
ran_out_of_space_ = true;
data_writer.second.writer->acknowledge_out_of_space();
}
+ RenameTempFile(data_writer.second.writer.get());
+ data_writer.second.writer.reset();
}
}
if (data_writer_) {
@@ -217,14 +229,16 @@
ran_out_of_space_ = true;
data_writer_->acknowledge_out_of_space();
}
+ RenameTempFile(data_writer_.get());
+ data_writer_.reset();
}
}
void MultiNodeLogNamer::OpenForwardedTimestampWriter(const Channel *channel,
DataWriter *data_writer) {
std::string filename =
- absl::StrCat(base_name_, "_timestamps", channel->name()->string_view(),
- "/", channel->type()->string_view(), ".part",
+ absl::StrCat("_timestamps", channel->name()->string_view(), "/",
+ channel->type()->string_view(), ".part",
data_writer->part_number, ".bfbs");
CreateBufferWriter(filename, &data_writer->writer);
}
@@ -232,33 +246,32 @@
void MultiNodeLogNamer::OpenWriter(const Channel *channel,
DataWriter *data_writer) {
const std::string filename = absl::StrCat(
- base_name_, "_", CHECK_NOTNULL(channel->source_node())->string_view(),
- "_data", channel->name()->string_view(), "/",
- channel->type()->string_view(), ".part", data_writer->part_number,
- ".bfbs");
+ "_", CHECK_NOTNULL(channel->source_node())->string_view(), "_data",
+ channel->name()->string_view(), "/", channel->type()->string_view(),
+ ".part", data_writer->part_number, ".bfbs");
CreateBufferWriter(filename, &data_writer->writer);
}
-std::unique_ptr<DetachedBufferWriter> MultiNodeLogNamer::OpenDataWriter() {
- std::string name = base_name_;
+void MultiNodeLogNamer::OpenDataWriter() {
+ std::string name;
if (node() != nullptr) {
name = absl::StrCat(name, "_", node()->name()->string_view());
}
- return std::make_unique<DetachedBufferWriter>(
- absl::StrCat(name, "_data.part", part_number_, ".bfbs"),
- std::make_unique<DummyEncoder>());
+ absl::StrAppend(&name, "_data.part", part_number_, ".bfbs");
+ CreateBufferWriter(name, &data_writer_);
}
void MultiNodeLogNamer::CreateBufferWriter(
- std::string_view filename,
- std::unique_ptr<DetachedBufferWriter> *destination) {
+ std::string_view path, std::unique_ptr<DetachedBufferWriter> *destination) {
if (ran_out_of_space_) {
// Refuse to open any new files, which might skip data. Any existing files
// are in the same folder, which means they're on the same filesystem, which
// means they're probably going to run out of space and get stuck too.
return;
}
+ const std::string filename = absl::StrCat(base_name_, path, temp_suffix_);
if (!destination->get()) {
+ all_filenames_.emplace_back(path);
*destination = std::make_unique<DetachedBufferWriter>(
filename, std::make_unique<DummyEncoder>());
return;
@@ -268,9 +281,31 @@
ran_out_of_space_ = true;
return;
}
+ RenameTempFile(destination->get());
+ all_filenames_.emplace_back(path);
*destination->get() =
DetachedBufferWriter(filename, std::make_unique<DummyEncoder>());
}
+void MultiNodeLogNamer::RenameTempFile(DetachedBufferWriter *destination) {
+ if (temp_suffix_.empty()) {
+ return;
+ }
+ const std::string current_filename = std::string(destination->filename());
+ CHECK(current_filename.size() > temp_suffix_.size());
+ const std::string final_filename =
+ current_filename.substr(0, current_filename.size() - temp_suffix_.size());
+ const int result = rename(current_filename.c_str(), final_filename.c_str());
+ if (result != 0) {
+ if (errno == ENOSPC) {
+ ran_out_of_space_ = true;
+ return;
+ } else {
+ PLOG(FATAL) << "Renaming " << current_filename << " to " << final_filename
+ << " failed";
+ }
+ }
+}
+
} // namespace logger
} // namespace aos
diff --git a/aos/events/logging/log_namer.h b/aos/events/logging/log_namer.h
index 661f28d..d9601ae 100644
--- a/aos/events/logging/log_namer.h
+++ b/aos/events/logging/log_namer.h
@@ -122,12 +122,27 @@
// Log namer which uses a config and a base name to name a bunch of files.
class MultiNodeLogNamer : public LogNamer {
public:
+ // If temp_suffix is set, then this will write files under names beginning
+ // with the specified suffix, and then rename them to the desired name after
+ // they are fully written.
+ //
+ // This is useful to enable incremental copying of the log files.
+ //
+ // Defaults to writing directly to the final filename.
MultiNodeLogNamer(std::string_view base_name,
- const Configuration *configuration, const Node *node);
- ~MultiNodeLogNamer() override = default;
+ const Configuration *configuration, const Node *node,
+ std::string_view temp_suffix = "");
+ ~MultiNodeLogNamer() override;
std::string_view base_name() const { return base_name_; }
+ // A list of all the filenames we've written.
+ //
+ // This only includes the part after base_name().
+ const std::vector<std::string> &all_filenames() const {
+ return all_filenames_;
+ }
+
void WriteHeader(
aos::SizePrefixedFlatbufferDetachedBuffer<LogFileHeader> *header,
const Node *node) override;
@@ -192,18 +207,22 @@
void OpenWriter(const Channel *channel, DataWriter *data_writer);
// Opens the main data writer file for this node responsible for data_writer_.
- std::unique_ptr<DetachedBufferWriter> OpenDataWriter();
+ void OpenDataWriter();
- void CreateBufferWriter(std::string_view filename,
+ void CreateBufferWriter(std::string_view path,
std::unique_ptr<DetachedBufferWriter> *destination);
+ void RenameTempFile(DetachedBufferWriter *destination);
+
const std::string base_name_;
+ const std::string temp_suffix_;
const Configuration *const configuration_;
const UUID uuid_;
size_t part_number_ = 0;
bool ran_out_of_space_ = false;
+ std::vector<std::string> all_filenames_;
// File to write both delivery timestamps and local data to.
std::unique_ptr<DetachedBufferWriter> data_writer_;