Copy flatbuffers efficiently and respecting DAGs.
Deduce the memory region needed to be copied by traversing the DAG, and
then copy it. This has no mallocs (yay!), and works with DAGs.
There are some cases where we actually want to copy things recursively.
Do that when asked.
Change-Id: Ia0ffde26d569fa92ee2bbb49706c17d9d657d125
diff --git a/aos/flatbuffer_merge_test.cc b/aos/flatbuffer_merge_test.cc
index 890a831..85773a3 100644
--- a/aos/flatbuffer_merge_test.cc
+++ b/aos/flatbuffer_merge_test.cc
@@ -1,5 +1,8 @@
#include "aos/flatbuffer_merge.h"
+#include <string_view>
+
+#include "absl/strings/escaping.h"
#include "gtest/gtest.h"
#include "aos/json_to_flatbuffer.h"
@@ -9,6 +12,20 @@
namespace aos {
namespace testing {
+std::string_view FromFbb(const flatbuffers::FlatBufferBuilder &fbb) {
+ return std::string_view(
+ reinterpret_cast<const char *>(fbb.GetCurrentBufferPointer()),
+ fbb.GetSize());
+}
+
+std::string_view FromFbb(const flatbuffers::DetachedBuffer &b) {
+ return std::string_view(reinterpret_cast<const char *>(b.data()), b.size());
+}
+
+absl::Span<const uint8_t> ToSpan(const flatbuffers::DetachedBuffer &b) {
+ return absl::Span<const uint8_t>(b.data(), b.size());
+}
+
class FlatbufferMerge : public ::testing::Test {
public:
FlatbufferMerge() {}
@@ -232,6 +249,103 @@
EXPECT_EQ(out_nested,
FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
}
+
+ // TODO(austin): Try more flatbuffers...
+ // Now try copying various flatbuffers and confirming they match.
+ {
+ flatbuffers::FlatBufferBuilder aos_flatbuffer_copy_fbb;
+ aos_flatbuffer_copy_fbb.ForceDefaults(true);
+
+ LOG(INFO) << "Copying " << in1 << " "
+ << absl::BytesToHexString(FromFbb(fb1)) << " at "
+ << reinterpret_cast<const void *>(fb1.data()) << " size "
+ << fb1.size();
+ aos_flatbuffer_copy_fbb.Finish(CopyFlatBuffer<Configuration>(
+ flatbuffers::GetRoot<Configuration>(fb1.data()),
+ &aos_flatbuffer_copy_fbb));
+
+ aos::FlatbufferDetachedBuffer<Configuration> fb_copy(
+ aos_flatbuffer_copy_fbb.Release());
+ ASSERT_NE(fb_copy.size(), 0u);
+
+ flatbuffers::Verifier v(fb1.data(), fb1.size());
+ EXPECT_TRUE(
+ v.VerifyTable(flatbuffers::GetRoot<Configuration>(fb1.data())));
+
+ ASSERT_TRUE(fb_copy.Verify()) << in1;
+
+ EXPECT_EQ(in1, FlatbufferToJson(fb_copy));
+ }
+
+ {
+ flatbuffers::FlatBufferBuilder aos_flatbuffer_copy_message_ptr_fbb;
+ aos_flatbuffer_copy_message_ptr_fbb.ForceDefaults(true);
+
+ aos_flatbuffer_copy_message_ptr_fbb.Finish(CopyFlatBuffer<Configuration>(
+ flatbuffers::GetRoot<Configuration>(fb2.data()),
+ &aos_flatbuffer_copy_message_ptr_fbb));
+
+ aos::FlatbufferDetachedBuffer<Configuration> fb_copy_message_ptr(
+ aos_flatbuffer_copy_message_ptr_fbb.Release());
+ ASSERT_NE(fb_copy_message_ptr.size(), 0u);
+
+ flatbuffers::FlatBufferBuilder aos_flatbuffer_copy_full_fbb;
+ aos_flatbuffer_copy_full_fbb.ForceDefaults(true);
+
+ aos_flatbuffer_copy_full_fbb.Finish(BlindCopyFlatBuffer<Configuration>(
+ aos::FlatbufferSpan<Configuration>(ToSpan(fb2)),
+ &aos_flatbuffer_copy_full_fbb));
+
+ aos::FlatbufferDetachedBuffer<Configuration> fb_copy_full(
+ aos_flatbuffer_copy_full_fbb.Release());
+ ASSERT_NE(fb_copy_full.size(), 0u);
+
+ flatbuffers::Verifier v(fb2.data(), fb2.size());
+ EXPECT_TRUE(
+ v.VerifyTable(flatbuffers::GetRoot<Configuration>(fb2.data())));
+
+ LOG(INFO) << "Verifying copy of " << in2;
+ ASSERT_TRUE(fb_copy_message_ptr.Verify()) << in2;
+ LOG(INFO) << "Verifying full of " << in2;
+ ASSERT_TRUE(fb_copy_full.Verify()) << in2;
+
+ EXPECT_EQ(in2, FlatbufferToJson(fb_copy_message_ptr));
+ EXPECT_EQ(in2, FlatbufferToJson(fb_copy_full));
+ }
+
+ {
+ flatbuffers::FlatBufferBuilder aos_flatbuffer_copy_fbb;
+ aos_flatbuffer_copy_fbb.ForceDefaults(true);
+
+ aos_flatbuffer_copy_fbb.Finish(CopyFlatBuffer<Configuration>(
+ flatbuffers::GetRoot<Configuration>(fb1_nested.data()),
+ &aos_flatbuffer_copy_fbb));
+
+ aos::FlatbufferDetachedBuffer<Configuration> fb_copy(
+ aos_flatbuffer_copy_fbb.Release());
+ ASSERT_NE(fb_copy.size(), 0u);
+
+ ASSERT_TRUE(fb_copy.Verify());
+
+ EXPECT_EQ(in1_nested, FlatbufferToJson(fb_copy));
+ }
+
+ {
+ flatbuffers::FlatBufferBuilder aos_flatbuffer_copy_fbb;
+ aos_flatbuffer_copy_fbb.ForceDefaults(true);
+
+ aos_flatbuffer_copy_fbb.Finish(CopyFlatBuffer<Configuration>(
+ flatbuffers::GetRoot<Configuration>(fb2_nested.data()),
+ &aos_flatbuffer_copy_fbb));
+
+ aos::FlatbufferDetachedBuffer<Configuration> fb_copy(
+ aos_flatbuffer_copy_fbb.Release());
+ ASSERT_NE(fb_copy.size(), 0u);
+
+ ASSERT_TRUE(fb_copy.Verify());
+
+ EXPECT_EQ(in2_nested, FlatbufferToJson(fb_copy));
+ }
}
};
@@ -342,6 +456,126 @@
"\"name\": \"woo2\" }, { \"name\": \"wo3\" } ] }");
}
+// Test nested messages, and arrays of nested messages.
+TEST(FlatbufferCopy, WholesaleCopy) {
+ flatbuffers::FlatBufferBuilder fbb_expected;
+ fbb_expected.ForceDefaults(true);
+ fbb_expected.DedupVtables(false);
+
+ {
+ flatbuffers::Offset<flatbuffers::String> name1_offset =
+ fbb_expected.CreateString("wow");
+
+ std::vector<flatbuffers::Offset<Application>> application_offsets;
+ Application::Builder a1_builder(fbb_expected);
+ a1_builder.add_name(name1_offset);
+ a1_builder.add_priority(7);
+ a1_builder.add_long_thingy(0x2549713132LL);
+ application_offsets.emplace_back(a1_builder.Finish());
+
+ flatbuffers::Offset<flatbuffers::String> name2_offset =
+ fbb_expected.CreateString("foo");
+ Application::Builder a2_builder(fbb_expected);
+ a2_builder.add_name(name2_offset);
+ a2_builder.add_priority(3);
+ a2_builder.add_long_thingy(9);
+ application_offsets.emplace_back(a2_builder.Finish());
+
+ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Application>>>
+ applications_offset = fbb_expected.CreateVector(application_offsets);
+
+ Configuration::Builder configuration_builder(fbb_expected);
+ configuration_builder.add_apps(applications_offset);
+ fbb_expected.Finish(configuration_builder.Finish());
+ }
+
+ LOG(INFO) << "Initial alignment " << fbb_expected.GetBufferMinAlignment();
+
+ aos::FlatbufferDetachedBuffer<Configuration> expected(fbb_expected.Release());
+
+ LOG(INFO) << "Expected "
+ << absl::BytesToHexString(std::string_view(
+ reinterpret_cast<char *>(expected.span().data()),
+ expected.span().size()));
+
+ aos::FlatbufferDetachedBuffer<Application> a1 = []() {
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(true);
+
+ flatbuffers::Offset<flatbuffers::String> name1_offset =
+ fbb.CreateString("wow");
+
+ Application::Builder a1_builder(fbb);
+ a1_builder.add_name(name1_offset);
+ a1_builder.add_priority(7);
+ a1_builder.add_long_thingy(0x2549713132LL);
+ flatbuffers::Offset<Application> a1 = a1_builder.Finish();
+
+ fbb.Finish(a1);
+ return fbb.Release();
+ }();
+
+ aos::FlatbufferDetachedBuffer<Application> a2 =
+ JsonToFlatbuffer<Application>(static_cast<const char *>(
+ "{ \"name\": \"foo\", \"priority\": 3, \"long_thingy\": 9 }"));
+
+ aos::FlatbufferDetachedBuffer<Configuration> c1 = [&a1, &a2]() {
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(true);
+
+ std::vector<flatbuffers::Offset<Application>> application_offsets;
+ application_offsets.emplace_back(BlindCopyFlatBuffer(a1, &fbb));
+ application_offsets.emplace_back(BlindCopyFlatBuffer(a2, &fbb));
+
+ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Application>>>
+ applications_offset = fbb.CreateVector(application_offsets);
+
+ Configuration::Builder configuration_builder(fbb);
+ configuration_builder.add_apps(applications_offset);
+ fbb.Finish(configuration_builder.Finish());
+
+ return fbb.Release();
+ }();
+
+ LOG(INFO) << "Got "
+ << absl::BytesToHexString(
+ std::string_view(reinterpret_cast<char *>(c1.span().data()),
+ c1.span().size()));
+
+ aos::FlatbufferDetachedBuffer<Configuration> c2 = [&a1, &a2]() {
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(true);
+
+ std::vector<flatbuffers::Offset<Application>> application_offsets;
+ application_offsets.emplace_back(CopyFlatBuffer(&a1.message(), &fbb));
+ application_offsets.emplace_back(CopyFlatBuffer(&a2.message(), &fbb));
+
+ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Application>>>
+ applications_offset = fbb.CreateVector(application_offsets);
+
+ Configuration::Builder configuration_builder(fbb);
+ configuration_builder.add_apps(applications_offset);
+ fbb.Finish(configuration_builder.Finish());
+
+ return fbb.Release();
+ }();
+
+ LOG(INFO) << "Got "
+ << absl::BytesToHexString(
+ std::string_view(reinterpret_cast<char *>(c2.span().data()),
+ c2.span().size()));
+
+ ASSERT_TRUE(expected.Verify());
+ ASSERT_TRUE(c1.Verify());
+ ASSERT_TRUE(c2.Verify());
+
+ LOG(INFO) << FlatbufferToJson(expected);
+ LOG(INFO) << FlatbufferToJson(c1);
+ LOG(INFO) << FlatbufferToJson(c2);
+ EXPECT_EQ(FlatbufferToJson(expected), FlatbufferToJson(c1));
+ EXPECT_EQ(FlatbufferToJson(expected), FlatbufferToJson(c2));
+}
+
// Tests a compare of 2 basic (different) messages.
TEST_F(FlatbufferMerge, CompareDifferent) {
aos::FlatbufferDetachedBuffer<Configuration> message1(JsonToFlatbuffer(