MMapping for flatbuffers
This lets us lazily load flatbuffers using mmap to both reduce memory
usage and to increase speed for use cases where we don't need to access
everything.
Change-Id: Ia271350b4c7cbb7a0a86ca094ef69c34716ba754
Signed-off-by: Austin Schuh <austin.schuh@bluerivertech.com>
diff --git a/aos/BUILD b/aos/BUILD
index 9049e08..8a8a19a 100644
--- a/aos/BUILD
+++ b/aos/BUILD
@@ -409,6 +409,7 @@
deps = [
"//aos:macros",
"//aos/containers:resizeable_buffer",
+ "//aos/util:file",
"@com_github_google_flatbuffers//:flatbuffers",
"@com_github_google_glog//:glog",
"@com_google_absl//absl/strings",
@@ -581,6 +582,7 @@
":json_to_flatbuffer",
":json_to_flatbuffer_fbs",
"//aos/testing:googletest",
+ "//aos/testing:tmpdir",
],
)
diff --git a/aos/flatbuffers.h b/aos/flatbuffers.h
index 68a8ad5..74d74ba 100644
--- a/aos/flatbuffers.h
+++ b/aos/flatbuffers.h
@@ -7,6 +7,7 @@
#include "absl/types/span.h"
#include "aos/containers/resizeable_buffer.h"
#include "aos/macros.h"
+#include "aos/util/file.h"
#include "flatbuffers/flatbuffers.h" // IWYU pragma: export
#include "glog/logging.h"
@@ -501,6 +502,33 @@
span.size());
}
+// MMap a flatbuffer on disk.
+template <typename T>
+class FlatbufferMMap : public NonSizePrefixedFlatbuffer<T> {
+ public:
+ // Builds a Flatbuffer by mmaping the data from a flatbuffer saved on disk.
+ FlatbufferMMap(const std::string &flatbuffer_path) {
+ span_ = util::MMapFile(flatbuffer_path);
+ }
+
+ // Copies the reference to the mapped memory.
+ FlatbufferMMap(const FlatbufferMMap &) = default;
+ FlatbufferMMap &operator=(const FlatbufferMMap<T> &other) = default;
+
+ // Moves the reference to the mapped memory from one pointer to another.
+ FlatbufferMMap(FlatbufferMMap &&) = default;
+ FlatbufferMMap &operator=(FlatbufferMMap<T> &&other) = default;
+
+ absl::Span<uint8_t> span() override {
+ LOG(FATAL) << "Unimplemented. A flatbuffer is immutable.";
+ return *span_;
+ }
+ absl::Span<const uint8_t> span() const override { return *span_; }
+
+ private:
+ std::shared_ptr<absl::Span<uint8_t>> span_;
+};
+
} // namespace aos
#endif // AOS_FLATBUFFERS_H_
diff --git a/aos/flatbuffers_test.cc b/aos/flatbuffers_test.cc
index e3030f1..f37af53 100644
--- a/aos/flatbuffers_test.cc
+++ b/aos/flatbuffers_test.cc
@@ -1,9 +1,10 @@
#include "aos/flatbuffers.h"
-#include "gtest/gtest.h"
-
+#include "absl/strings/str_cat.h"
#include "aos/json_to_flatbuffer.h"
#include "aos/json_to_flatbuffer_generated.h"
+#include "aos/testing/tmpdir.h"
+#include "gtest/gtest.h"
namespace aos {
namespace testing {
@@ -21,5 +22,38 @@
EXPECT_FALSE(empty.Verify());
}
+// Tests the ability to map a flatbuffer on disk to memory
+TEST(FlatbufferMMapTest, Verify) {
+ FlatbufferDetachedBuffer<Configuration> fb =
+ JsonToFlatbuffer<Configuration>("{\"foo_int\": 3}");
+
+ const std::string fb_path = absl::StrCat(TestTmpDir(), "/fb.bfbs");
+ WriteFlatbufferToFile(fb_path, fb);
+
+ FlatbufferMMap<Configuration> fb_mmap(fb_path);
+ EXPECT_TRUE(fb.Verify());
+ EXPECT_TRUE(fb_mmap.Verify());
+ ASSERT_EQ(fb_mmap.message().foo_int(), 3);
+
+ // Verify that copying works
+ {
+ FlatbufferMMap<Configuration> fb_mmap2(fb_path);
+ fb_mmap2 = fb_mmap;
+ EXPECT_TRUE(fb_mmap.Verify());
+ EXPECT_TRUE(fb_mmap2.Verify());
+ ASSERT_EQ(fb_mmap2.message().foo_int(), 3);
+ ASSERT_EQ(fb_mmap.message().foo_int(), 3);
+ }
+ EXPECT_TRUE(fb_mmap.Verify());
+ ASSERT_EQ(fb_mmap.message().foo_int(), 3);
+
+ // Verify that moving works
+ {
+ FlatbufferMMap<Configuration> fb_mmap3(fb_path);
+ fb_mmap3 = std::move(fb_mmap);
+ EXPECT_TRUE(fb_mmap3.Verify());
+ ASSERT_EQ(fb_mmap3.message().foo_int(), 3);
+ }
+}
} // namespace testing
} // namespace aos
diff --git a/aos/util/BUILD b/aos/util/BUILD
index 59dec18..daacba1 100644
--- a/aos/util/BUILD
+++ b/aos/util/BUILD
@@ -246,6 +246,7 @@
"//aos/scoped:scoped_fd",
"@com_github_google_glog//:glog",
"@com_google_absl//absl/strings",
+ "@com_google_absl//absl/types:span",
],
)
diff --git a/aos/util/file.cc b/aos/util/file.cc
index 4473d4d..cec5a6e 100644
--- a/aos/util/file.cc
+++ b/aos/util/file.cc
@@ -2,6 +2,7 @@
#include <fcntl.h>
#include <fts.h>
+#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
@@ -151,5 +152,24 @@
}
}
+std::shared_ptr<absl::Span<uint8_t>> MMapFile(const std::string &path) {
+ int fd = open(path.c_str(), O_RDONLY);
+ PCHECK(fd != -1) << "Unable to open file " << path;
+ struct stat sb;
+ PCHECK(fstat(fd, &sb) != -1) << ": Unable to get file size of " << path;
+ uint8_t *start = reinterpret_cast<uint8_t *>(
+ mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0));
+ CHECK(start != MAP_FAILED) << ": Unable to open mapping to file " << path;
+ std::shared_ptr<absl::Span<uint8_t>> span =
+ std::shared_ptr<absl::Span<uint8_t>>(
+ new absl::Span<uint8_t>(start, sb.st_size),
+ [](absl::Span<uint8_t> *span) {
+ PCHECK(munmap(span->data(), span->size()) != -1);
+ delete span;
+ });
+ close(fd);
+ return span;
+}
+
} // namespace util
} // namespace aos
diff --git a/aos/util/file.h b/aos/util/file.h
index 8089225..7c06ece 100644
--- a/aos/util/file.h
+++ b/aos/util/file.h
@@ -2,9 +2,12 @@
#define AOS_UTIL_FILE_H_
#include <sys/stat.h>
+
+#include <memory>
#include <string>
#include <string_view>
+#include "absl/types/span.h"
#include "glog/logging.h"
namespace aos {
@@ -32,6 +35,9 @@
// runs across.
void UnlinkRecursive(std::string_view path);
+// Maps file from disk into memory
+std::shared_ptr<absl::Span<uint8_t>> MMapFile(const std::string &path);
+
} // namespace util
} // namespace aos