Merge changes Ie8fcff1e,Ief9a9a2d,Ie5186482,Ia277200e,Idd29395f, ...
* changes:
Add a Flatbuffers containing object, and more variants
Add support for has_foo and clear_foo to flatbuffers.
Add support for absl::string_view to flatbuffers
Add comment support in JSON.
Handle multiple arrays.
Add a flatbuffer merge function
diff --git a/aos/BUILD b/aos/BUILD
index 95418ea..eba1b45 100644
--- a/aos/BUILD
+++ b/aos/BUILD
@@ -440,9 +440,11 @@
hdrs = ["json_to_flatbuffer.h"],
deps = [
":flatbuffer_utils",
+ ":flatbuffers",
":json_tokenizer",
"//aos/logging",
"@com_github_google_flatbuffers//:flatbuffers",
+ "@com_github_google_glog//:glog",
"@com_google_absl//absl/strings",
],
)
@@ -458,3 +460,38 @@
"//aos/testing:googletest",
],
)
+
+cc_library(
+ name = "flatbuffer_merge",
+ srcs = ["flatbuffer_merge.cc"],
+ hdrs = ["flatbuffer_merge.h"],
+ copts = ["-Wno-cast-align"],
+ deps = [
+ ":flatbuffer_utils",
+ ":flatbuffers",
+ "@com_github_google_flatbuffers//:flatbuffers",
+ ],
+)
+
+cc_test(
+ name = "flatbuffer_merge_test",
+ srcs = [
+ "flatbuffer_merge_test.cc",
+ ],
+ deps = [
+ ":flatbuffer_merge",
+ ":json_to_flatbuffer",
+ ":json_to_flatbuffer_flatbuffer",
+ "//aos/testing:googletest",
+ ],
+)
+
+cc_library(
+ name = "flatbuffers",
+ hdrs = [
+ "flatbuffers.h",
+ ],
+ deps = [
+ "@com_github_google_flatbuffers//:flatbuffers",
+ ],
+)
diff --git a/aos/flatbuffer_merge.cc b/aos/flatbuffer_merge.cc
new file mode 100644
index 0000000..c8cc742
--- /dev/null
+++ b/aos/flatbuffer_merge.cc
@@ -0,0 +1,529 @@
+#include "aos/flatbuffer_merge.h"
+
+#include <stdio.h>
+
+#include "aos/flatbuffer_utils.h"
+#include "flatbuffers/flatbuffers.h"
+#include "flatbuffers/minireflect.h"
+
+namespace aos {
+
+namespace {
+
+// Simple structure to hold both field_offsets and elements.
+struct OffsetAndFieldOffset {
+ OffsetAndFieldOffset(flatbuffers::voffset_t new_field_offset,
+ flatbuffers::Offset<flatbuffers::String> new_element)
+ : field_offset(new_field_offset), element(new_element) {}
+ OffsetAndFieldOffset(flatbuffers::voffset_t new_field_offset,
+ flatbuffers::Offset<flatbuffers::Table> new_element)
+ : field_offset(new_field_offset), element(new_element.o) {}
+
+ flatbuffers::voffset_t field_offset;
+ flatbuffers::Offset<flatbuffers::String> element;
+};
+
+// Merges a single element to a builder for the provided field.
+// One or both of t1 and t2 must be non-null. If one is null, this method
+// copies instead of merging.
+template <typename T>
+void MergeElement(flatbuffers::voffset_t field_offset,
+ const flatbuffers::Table *t1, const flatbuffers::Table *t2,
+ flatbuffers::FlatBufferBuilder *fbb) {
+ const uint8_t *val1 =
+ t1 != nullptr ? t1->GetAddressOf(field_offset) : nullptr;
+ const uint8_t *val2 =
+ t2 != nullptr ? t2->GetAddressOf(field_offset) : nullptr;
+ const bool t1_has = val1 != nullptr;
+ const bool t2_has = val2 != nullptr;
+
+ if (t2_has) {
+ fbb->AddElement<T>(field_offset, flatbuffers::ReadScalar<T>(val2), 0);
+ } else if (t1_has) {
+ fbb->AddElement<T>(field_offset, flatbuffers::ReadScalar<T>(val1), 0);
+ }
+}
+
+// Merges a single string to a builder for the provided field.
+// One or both of t1 and t2 must be non-null. If one is null, this method
+// copies instead of merging.
+void MergeString(flatbuffers::voffset_t field_offset,
+ const flatbuffers::Table *t1, const flatbuffers::Table *t2,
+ flatbuffers::FlatBufferBuilder *fbb,
+ ::std::vector<OffsetAndFieldOffset> *elements) {
+ const uint8_t *val1 =
+ t1 != nullptr ? t1->GetAddressOf(field_offset) : nullptr;
+ const uint8_t *val2 =
+ t2 != nullptr ? t2->GetAddressOf(field_offset) : nullptr;
+ const bool t1_has = val1 != nullptr;
+ const bool t2_has = val2 != nullptr;
+
+ if (t2_has) {
+ val2 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val2);
+ const flatbuffers::String *string2 =
+ reinterpret_cast<const flatbuffers::String *>(val2);
+ elements->emplace_back(field_offset,
+ fbb->CreateString(string2->data(), string2->size()));
+ } else if (t1_has) {
+ val1 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val1);
+ const flatbuffers::String *string1 =
+ reinterpret_cast<const flatbuffers::String *>(val1);
+ elements->emplace_back(field_offset,
+ fbb->CreateString(string1->data(), string1->size()));
+ }
+}
+
+// Merges an object to a builder for the provided field.
+// One or both of t1 and t2 must be non-null. If one is null, this method
+// copies instead of merging.
+void MergeTables(flatbuffers::voffset_t field_offset,
+ const flatbuffers::Table *t1, const flatbuffers::Table *t2,
+ const flatbuffers::TypeTable *sub_typetable,
+ flatbuffers::FlatBufferBuilder *fbb,
+ ::std::vector<OffsetAndFieldOffset> *elements) {
+ const uint8_t *val1 =
+ t1 != nullptr ? t1->GetAddressOf(field_offset) : nullptr;
+ const uint8_t *val2 =
+ t2 != nullptr ? t2->GetAddressOf(field_offset) : nullptr;
+ const bool t1_has = val1 != nullptr;
+ const bool t2_has = val2 != nullptr;
+ if (t1_has || t2_has) {
+ if (val1 != nullptr) {
+ val1 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val1);
+ }
+ if (val2 != nullptr) {
+ val2 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val2);
+ }
+
+ const flatbuffers::Table *sub_t1 =
+ reinterpret_cast<const flatbuffers::Table *>(val1);
+ const flatbuffers::Table *sub_t2 =
+ reinterpret_cast<const flatbuffers::Table *>(val2);
+
+ elements->emplace_back(field_offset,
+ MergeFlatBuffers(sub_typetable, sub_t1, sub_t2, fbb));
+ }
+}
+
+// Adds a vector of strings to the elements vector so it can be added later.
+// One or both of t1 and t2 must be non-null. If one is null, this method
+// copies instead of merging.
+void AddVectorOfStrings(flatbuffers::ElementaryType elementary_type,
+ flatbuffers::voffset_t field_offset,
+ const flatbuffers::Table *t1,
+ const flatbuffers::Table *t2,
+ flatbuffers::FlatBufferBuilder *fbb,
+ ::std::vector<OffsetAndFieldOffset> *elements) {
+ const uint8_t *val1 =
+ t1 != nullptr ? t1->GetAddressOf(field_offset) : nullptr;
+ const uint8_t *val2 =
+ t2 != nullptr ? t2->GetAddressOf(field_offset) : nullptr;
+ const bool t1_has = val1 != nullptr;
+ const bool t2_has = val2 != nullptr;
+
+ // Compute end size of the vector.
+ size_t size = 0;
+ if (t1_has) {
+ val1 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val1);
+ auto vec1 = reinterpret_cast<
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *>(
+ val1);
+ size += vec1->size();
+ }
+ if (t2_has) {
+ val2 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val2);
+ auto vec2 = reinterpret_cast<
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *>(
+ val2);
+ size += vec2->size();
+ }
+
+ // Only add the vector if there is something to add.
+ if (t1_has || t2_has) {
+ const size_t inline_size =
+ flatbuffers::InlineSize(elementary_type, nullptr);
+
+ ::std::vector<flatbuffers::Offset<flatbuffers::String>> string_elements;
+
+ // Pack the contents in in reverse order.
+ if (t2_has) {
+ auto vec2 = reinterpret_cast<const flatbuffers::Vector<
+ flatbuffers::Offset<flatbuffers::String>> *>(val2);
+ for (auto i = vec2->rbegin(); i != vec2->rend(); ++i) {
+ const flatbuffers::String *s = *i;
+ string_elements.emplace_back(fbb->CreateString(s->data(), s->size()));
+ }
+ }
+ if (t1_has) {
+ auto vec1 = reinterpret_cast<const flatbuffers::Vector<
+ flatbuffers::Offset<flatbuffers::String>> *>(val1);
+ for (auto i = vec1->rbegin(); i != vec1->rend(); ++i) {
+ const flatbuffers::String *s = *i;
+ string_elements.emplace_back(fbb->CreateString(s->data(), s->size()));
+ }
+ }
+
+ // Start the vector.
+ fbb->StartVector(size, inline_size);
+
+ for (const flatbuffers::Offset<flatbuffers::String> &element :
+ string_elements) {
+ fbb->PushElement(element);
+ }
+
+ // And then finish the vector and put it in the list of offsets to add to
+ // the message when it finishes.
+ elements->emplace_back(
+ field_offset,
+ flatbuffers::Offset<flatbuffers::String>(fbb->EndVector(size)));
+ }
+}
+
+// Adds a vector of values to the elements vector so it can be added later.
+// One or both of t1 and t2 must be non-null. If one is null, this method
+// copies instead of merging.
+template <typename T>
+void AddVector(flatbuffers::ElementaryType elementary_type,
+ flatbuffers::voffset_t field_offset,
+ const flatbuffers::Table *t1, const flatbuffers::Table *t2,
+ flatbuffers::FlatBufferBuilder *fbb,
+ ::std::vector<OffsetAndFieldOffset> *elements) {
+ const uint8_t *val1 =
+ t1 != nullptr ? t1->GetAddressOf(field_offset) : nullptr;
+ const uint8_t *val2 =
+ t2 != nullptr ? t2->GetAddressOf(field_offset) : nullptr;
+ const bool t1_has = val1 != nullptr;
+ const bool t2_has = val2 != nullptr;
+
+ // Compute end size of the vector.
+ size_t size = 0;
+ if (t1_has) {
+ val1 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val1);
+ auto vec1 = reinterpret_cast<const flatbuffers::Vector<T> *>(val1);
+ size += vec1->size();
+ }
+ if (t2_has) {
+ val2 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val2);
+ auto vec2 = reinterpret_cast<const flatbuffers::Vector<T> *>(val2);
+ size += vec2->size();
+ }
+
+ // Only add the vector if there is something to add.
+ if (t1_has || t2_has) {
+ const size_t inline_size =
+ flatbuffers::InlineSize(elementary_type, nullptr);
+
+ // Start the vector.
+ fbb->StartVector(size, inline_size);
+
+ // Pack the contents in in reverse order.
+ if (t2_has) {
+ auto vec2 = reinterpret_cast<const flatbuffers::Vector<T> *>(val2);
+ // Iterate backwards.
+ for (auto i = vec2->rbegin(); i != vec2->rend(); ++i) {
+ fbb->PushElement<T>(*i);
+ }
+ }
+ if (t1_has) {
+ auto vec1 = reinterpret_cast<const flatbuffers::Vector<T> *>(val1);
+ // Iterate backwards.
+ for (auto i = vec1->rbegin(); i != vec1->rend(); ++i) {
+ fbb->PushElement<T>(*i);
+ }
+ }
+ // And then finish the vector and put it in the list of offsets to add to
+ // the message when it finishes.
+ elements->emplace_back(
+ field_offset,
+ flatbuffers::Offset<flatbuffers::String>(fbb->EndVector(size)));
+ }
+}
+
+void AddVectorOfObjects(flatbuffers::FlatBufferBuilder *fbb,
+ ::std::vector<OffsetAndFieldOffset> *elements,
+ flatbuffers::ElementaryType elementary_type,
+ const flatbuffers::TypeTable *sub_typetable,
+ flatbuffers::voffset_t field_offset,
+ const flatbuffers::Table *t1,
+ const flatbuffers::Table *t2) {
+ const uint8_t *val1 =
+ t1 != nullptr ? t1->GetAddressOf(field_offset) : nullptr;
+ const uint8_t *val2 =
+ t2 != nullptr ? t2->GetAddressOf(field_offset) : nullptr;
+ const bool t1_has = val1 != nullptr;
+ const bool t2_has = val2 != nullptr;
+
+ // Compute end size of the vector.
+ size_t size = 0;
+ if (t1_has) {
+ val1 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val1);
+ auto vec1 = reinterpret_cast<
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::Table>> *>(
+ val1);
+ size += vec1->size();
+ }
+ if (t2_has) {
+ val2 += flatbuffers::ReadScalar<flatbuffers::uoffset_t>(val2);
+ auto vec2 = reinterpret_cast<
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::Table>> *>(
+ val2);
+ size += vec2->size();
+ }
+
+ // Only add the vector if there is something to add.
+ if (t1_has || t2_has) {
+ const size_t inline_size =
+ flatbuffers::InlineSize(elementary_type, sub_typetable);
+
+ ::std::vector<flatbuffers::Offset<flatbuffers::Table>> object_elements;
+
+ // Pack the contents in in reverse order.
+ if (t2_has) {
+ auto vec2 = reinterpret_cast<
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::Table>> *>(
+ val2);
+ for (auto i = vec2->rbegin(); i != vec2->rend(); ++i) {
+ const flatbuffers::Table *t = *i;
+
+ flatbuffers::Offset<flatbuffers::Table> end =
+ MergeFlatBuffers(sub_typetable, t, nullptr, fbb);
+
+ object_elements.emplace_back(end);
+ }
+ }
+ if (t1_has) {
+ auto vec1 = reinterpret_cast<
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::Table>> *>(
+ val1);
+ for (auto i = vec1->rbegin(); i != vec1->rend(); ++i) {
+ const flatbuffers::Table *t = *i;
+
+ flatbuffers::Offset<flatbuffers::Table> end =
+ MergeFlatBuffers(sub_typetable, t, nullptr, fbb);
+
+ object_elements.emplace_back(end);
+ }
+ }
+
+ // Start the vector.
+ fbb->StartVector(size, inline_size);
+
+ for (const flatbuffers::Offset<flatbuffers::Table> &element :
+ object_elements) {
+ fbb->PushElement(element);
+ }
+
+ // And then finish the vector and put it in the list of offsets to add to
+ // the message when it finishes.
+ elements->emplace_back(
+ field_offset,
+ flatbuffers::Offset<flatbuffers::String>(fbb->EndVector(size)));
+ }
+}
+
+} // namespace
+
+flatbuffers::Offset<flatbuffers::Table> MergeFlatBuffers(
+ const flatbuffers::TypeTable *typetable, const flatbuffers::Table *t1,
+ const flatbuffers::Table *t2, flatbuffers::FlatBufferBuilder *fbb) {
+ ::std::vector<OffsetAndFieldOffset> elements;
+
+ // We need to do this in 2 passes
+ // The first pass builds up all the sub-objects which are encoded in the
+ // message as offsets.
+ // The second pass builds up the actual table by adding all the values to the
+ // messages, and encoding the offsets in the table.
+ for (size_t field_index = 0; field_index < typetable->num_elems;
+ ++field_index) {
+ const flatbuffers::TypeCode type_code = typetable->type_codes[field_index];
+ const flatbuffers::ElementaryType elementary_type =
+ static_cast<flatbuffers::ElementaryType>(type_code.base_type);
+
+ const flatbuffers::voffset_t field_offset = flatbuffers::FieldIndexToOffset(
+ static_cast<flatbuffers::voffset_t>(field_index));
+
+ switch (elementary_type) {
+ case flatbuffers::ElementaryType::ET_UTYPE:
+ if (!type_code.is_vector) continue;
+ printf("ET_UTYPE, %s\n", typetable->names[field_index]);
+ break;
+ case flatbuffers::ElementaryType::ET_BOOL:
+ if (!type_code.is_vector) continue;
+ AddVector<uint8_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_CHAR:
+ if (!type_code.is_vector) continue;
+ AddVector<int8_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_UCHAR:
+ if (!type_code.is_vector) continue;
+ AddVector<uint8_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_SHORT:
+ if (!type_code.is_vector) continue;
+ AddVector<int16_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_USHORT:
+ if (!type_code.is_vector) continue;
+ AddVector<uint16_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_INT:
+ if (!type_code.is_vector) continue;
+ AddVector<int32_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_UINT:
+ if (!type_code.is_vector) continue;
+ AddVector<uint32_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_LONG:
+ if (!type_code.is_vector) continue;
+ AddVector<int64_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_ULONG:
+ if (!type_code.is_vector) continue;
+ AddVector<uint64_t>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_FLOAT:
+ if (!type_code.is_vector) continue;
+ AddVector<float>(elementary_type, field_offset, t1, t2, fbb, &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_DOUBLE:
+ if (!type_code.is_vector) continue;
+ AddVector<double>(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ break;
+ case flatbuffers::ElementaryType::ET_STRING:
+ if (!type_code.is_vector) {
+ MergeString(field_offset, t1, t2, fbb, &elements);
+ } else {
+ AddVectorOfStrings(elementary_type, field_offset, t1, t2, fbb,
+ &elements);
+ }
+ break;
+ case flatbuffers::ElementaryType::ET_SEQUENCE: {
+ const flatbuffers::TypeTable *sub_typetable =
+ typetable->type_refs[type_code.sequence_ref]();
+ if (!type_code.is_vector) {
+ MergeTables(field_offset, t1, t2, sub_typetable, fbb, &elements);
+ } else {
+ const flatbuffers::TypeTable *sub_typetable =
+ typetable->type_refs[type_code.sequence_ref]();
+
+ AddVectorOfObjects(fbb, &elements, elementary_type, sub_typetable,
+ field_offset, t1, t2);
+ }
+ } break;
+ }
+ }
+
+ const flatbuffers::uoffset_t start = fbb->StartTable();
+
+ // We want to do this the same way as the json library. Rip through the
+ // fields and generate a list of things to add. Then add them.
+ // Also need recursion for subtypes.
+ for (size_t field_index = 0; field_index < typetable->num_elems;
+ ++field_index) {
+ const flatbuffers::TypeCode type_code = typetable->type_codes[field_index];
+ if (type_code.is_vector) {
+ continue;
+ }
+ const flatbuffers::ElementaryType elementary_type =
+ static_cast<flatbuffers::ElementaryType>(type_code.base_type);
+
+ const flatbuffers::voffset_t field_offset = flatbuffers::FieldIndexToOffset(
+ static_cast<flatbuffers::voffset_t>(field_index));
+
+ switch (elementary_type) {
+ case flatbuffers::ElementaryType::ET_UTYPE:
+ // TODO(austin): Need to see one and try it.
+ printf("ET_UTYPE, %s\n", typetable->names[field_index]);
+ break;
+ case flatbuffers::ElementaryType::ET_BOOL: {
+ MergeElement<uint8_t>(field_offset, t1, t2, fbb);
+ } break;
+ case flatbuffers::ElementaryType::ET_CHAR:
+ MergeElement<int8_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_UCHAR:
+ MergeElement<uint8_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_SHORT:
+ MergeElement<int16_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_USHORT:
+ MergeElement<uint16_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_INT:
+ MergeElement<int32_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_UINT:
+ MergeElement<uint32_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_LONG:
+ MergeElement<int64_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_ULONG:
+ MergeElement<uint64_t>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_FLOAT:
+ MergeElement<float>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_DOUBLE:
+ MergeElement<double>(field_offset, t1, t2, fbb);
+ break;
+ case flatbuffers::ElementaryType::ET_STRING:
+ case flatbuffers::ElementaryType::ET_SEQUENCE:
+ // Already handled above since this is an uoffset.
+ break;
+ }
+ }
+
+ // And there is no need to check for duplicates since we are creating this
+ // list very carefully from the type table.
+ for (const OffsetAndFieldOffset &element : elements) {
+ fbb->AddOffset(element.field_offset, element.element);
+ }
+
+ return fbb->EndTable(start);
+}
+
+flatbuffers::Offset<flatbuffers::Table> MergeFlatBuffers(
+ const flatbuffers::TypeTable *typetable, const uint8_t *data1,
+ const uint8_t *data2, flatbuffers::FlatBufferBuilder *fbb) {
+ // Grab the 2 tables.
+ const flatbuffers::Table *t1 =
+ data1 != nullptr ? flatbuffers::GetRoot<flatbuffers::Table>(data1)
+ : nullptr;
+ const flatbuffers::Table *t2 =
+ data2 != nullptr ? flatbuffers::GetRoot<flatbuffers::Table>(data2)
+ : nullptr;
+
+ // And then do the actual build. This doesn't contain finish so we can nest
+ // them nicely.
+ return flatbuffers::Offset<flatbuffers::Table>(
+ MergeFlatBuffers(typetable, t1, t2, fbb));
+}
+
+flatbuffers::DetachedBuffer MergeFlatBuffers(
+ const flatbuffers::TypeTable *typetable, const uint8_t *data1,
+ const uint8_t *data2) {
+ // Build up a builder.
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(1);
+
+ // Finish up the buffer and return it.
+ fbb.Finish(MergeFlatBuffers(typetable, data1, data2, &fbb));
+
+ return fbb.Release();
+}
+
+} // namespace aos
diff --git a/aos/flatbuffer_merge.h b/aos/flatbuffer_merge.h
new file mode 100644
index 0000000..16372ed
--- /dev/null
+++ b/aos/flatbuffer_merge.h
@@ -0,0 +1,76 @@
+#ifndef AOS_FLATBUFFER_MERGE_H_
+#define AOS_FLATBUFFER_MERGE_H_
+
+#include <cstddef>
+#include <string>
+
+#include "aos/flatbuffers.h"
+#include "flatbuffers/flatbuffers.h"
+
+namespace aos {
+
+flatbuffers::DetachedBuffer MergeFlatBuffers(
+ const flatbuffers::TypeTable *typetable, const uint8_t *data1,
+ const uint8_t *data2);
+
+// Merges 2 flat buffers with the provided type table into the builder. Returns
+// the offset to the flatbuffers.
+// One or both of t1 and t2 must be non-null. If one is null, this method
+// coppies instead of merging.
+flatbuffers::Offset<flatbuffers::Table> MergeFlatBuffers(
+ const flatbuffers::TypeTable *typetable, const flatbuffers::Table *t1,
+ const flatbuffers::Table *t2, flatbuffers::FlatBufferBuilder *fbb);
+
+template <class T>
+inline flatbuffers::Offset<T> MergeFlatBuffers(
+ const flatbuffers::Table *t1,
+ const flatbuffers::Table *t2, flatbuffers::FlatBufferBuilder *fbb) {
+ return MergeFlatBuffers(T::MiniReflectTypeTable(), t1, t2, fbb).o;
+}
+
+template <class T>
+inline flatbuffers::DetachedBuffer MergeFlatBuffers(const uint8_t *data1,
+ const uint8_t *data2) {
+ return MergeFlatBuffers(T::MiniReflectTypeTable(), data1, data2);
+}
+
+template <class T>
+inline flatbuffers::DetachedBuffer MergeFlatBuffers(
+ const flatbuffers::DetachedBuffer &data1,
+ const flatbuffers::DetachedBuffer &data2) {
+ return MergeFlatBuffers(T::MiniReflectTypeTable(), data1.data(),
+ data2.data());
+}
+
+template <class T>
+inline aos::Flatbuffer<T> MergeFlatBuffers(const aos::Flatbuffer<T> &fb1,
+ const aos::Flatbuffer<T> &fb2) {
+ return aos::Flatbuffer<T>(
+ MergeFlatBuffers(T::MiniReflectTypeTable(), fb1.data(), fb2.data()));
+}
+
+template <class T>
+inline aos::Flatbuffer<T> MergeFlatBuffers(const T *fb1, const T *fb2) {
+ return aos::Flatbuffer<T>(MergeFlatBuffers(
+ T::MiniReflectTypeTable(), reinterpret_cast<const uint8_t *>(fb1),
+ reinterpret_cast<const uint8_t *>(fb2)));
+}
+
+template <class T>
+inline flatbuffers::Offset<T> CopyFlatBuffer(
+ const T *t1, flatbuffers::FlatBufferBuilder *fbb) {
+ return MergeFlatBuffers<T>(reinterpret_cast<const flatbuffers::Table *>(t1),
+ nullptr, fbb);
+}
+
+template <class T>
+inline Flatbuffer<T> CopyFlatBuffer(const T *t) {
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(1);
+ fbb.Finish(CopyFlatBuffer<T>(t, &fbb));
+ return Flatbuffer<T>(fbb.Release());
+}
+
+} // namespace aos
+
+#endif // AOS_FLATBUFFER_MERGE_H_
diff --git a/aos/flatbuffer_merge_test.cc b/aos/flatbuffer_merge_test.cc
new file mode 100644
index 0000000..ca63249
--- /dev/null
+++ b/aos/flatbuffer_merge_test.cc
@@ -0,0 +1,307 @@
+#include "aos/flatbuffer_merge.h"
+
+#include "gtest/gtest.h"
+
+#include "aos/json_to_flatbuffer.h"
+#include "aos/json_to_flatbuffer_generated.h"
+#include "flatbuffers/minireflect.h"
+
+namespace aos {
+namespace testing {
+
+class FlatbufferMerge : public ::testing::Test {
+ public:
+ FlatbufferMerge() {}
+
+ void JsonMerge(const ::std::string in1, const ::std::string in2,
+ const ::std::string out) {
+ printf("Merging: %s\n", in1.c_str());
+ printf("Merging: %s\n", in2.c_str());
+ const flatbuffers::DetachedBuffer fb1 = JsonToFlatbuffer(
+ static_cast<const char *>(in1.c_str()), ConfigurationTypeTable());
+
+ const ::std::string in1_nested = "{ \"nested_config\": " + in1 + " }";
+ const flatbuffers::DetachedBuffer fb1_nested =
+ JsonToFlatbuffer(static_cast<const char *>(in1_nested.c_str()),
+ ConfigurationTypeTable());
+
+ const flatbuffers::DetachedBuffer fb2 = JsonToFlatbuffer(
+ static_cast<const char *>(in2.c_str()), ConfigurationTypeTable());
+
+ const ::std::string in2_nested = "{ \"nested_config\": " + in2 + " }";
+ const flatbuffers::DetachedBuffer fb2_nested =
+ JsonToFlatbuffer(static_cast<const char *>(in2_nested.c_str()),
+ ConfigurationTypeTable());
+
+ const ::std::string out_nested = "{ \"nested_config\": " + out + " }";
+
+ const flatbuffers::DetachedBuffer empty =
+ JsonToFlatbuffer("{ }", ConfigurationTypeTable());
+
+ ASSERT_NE(fb1.size(), 0u);
+ ASSERT_NE(fb2.size(), 0u);
+ ASSERT_NE(fb1_nested.size(), 0u);
+ ASSERT_NE(fb2_nested.size(), 0u);
+
+ // We now want to run 7 tests.
+ // in1 merged "" -> in1.
+ // in2 merged "" -> in2.
+ // "" merged in1 -> in1.
+ // "" merged in2 -> in2.
+ // nullptr merged in1 -> in1.
+ // in1 merged nullptr -> in1.
+ // in1 merged in2 -> out.
+
+ {
+ // in1 merged with "" => in1.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb1.data(), empty.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1, FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // in2 merged with "" => in2.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb2.data(), empty.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in2, FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // "" merged with in1 => in1.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(empty.data(), fb1.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1, FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // "" merged with in2 => in2.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(empty.data(), fb2.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in2, FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // nullptr merged with in1 => in1.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(nullptr, fb1.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1, FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // in1 merged with nullptr => in1.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb1.data(), nullptr);
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1, FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // in1 merged with in2 => out.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb1.data(), fb2.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ const ::std::string merged_output =
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable());
+ EXPECT_EQ(out, merged_output);
+
+ printf("Merged to: %s\n", merged_output.c_str());
+ }
+
+ // Now, to make things extra exciting, nest a config inside a config. And
+ // run all the tests. This will exercise some fun nested merges and copies.
+
+ {
+ // in1_nested merged with "" => in1.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb1_nested.data(), empty.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // in2_nested merged with "" => in2_nested.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb2_nested.data(), empty.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in2_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // "" merged with in1_nested => in1_nested.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(empty.data(), fb1_nested.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // "" merged with in2_nested => in2_nested.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(empty.data(), fb2_nested.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in2_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // nullptr merged with in1_nested => in1_nested.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(nullptr, fb1_nested.data());
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // nullptr merged with in1_nested => in1_nested.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb1_nested.data(), nullptr);
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(in1_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+
+ {
+ // in1_nested merged with in2_nested => out_nested.
+ flatbuffers::DetachedBuffer fb_merged =
+ MergeFlatBuffers<Configuration>(fb1_nested, fb2_nested);
+ ASSERT_NE(fb_merged.size(), 0u);
+
+ EXPECT_EQ(out_nested,
+ FlatbufferToJson(fb_merged, ConfigurationTypeTable()));
+ }
+ }
+};
+
+// Test the easy ones. Test every type, single, no nesting.
+TEST_F(FlatbufferMerge, Basic) {
+ JsonMerge("{ \"foo_bool\": true }", "{ \"foo_bool\": false }",
+ "{ \"foo_bool\": false }");
+
+ JsonMerge("{ \"foo_bool\": false }", "{ \"foo_bool\": true }",
+ "{ \"foo_bool\": true }");
+
+ JsonMerge("{ \"foo_byte\": 5 }", "{ \"foo_byte\": -7 }",
+ "{ \"foo_byte\": -7 }");
+
+ JsonMerge("{ \"foo_ubyte\": 5 }", "{ \"foo_ubyte\": 7 }",
+ "{ \"foo_ubyte\": 7 }");
+
+ JsonMerge("{ \"foo_short\": 5 }", "{ \"foo_short\": 7 }",
+ "{ \"foo_short\": 7 }");
+
+ JsonMerge("{ \"foo_int\": 5 }", "{ \"foo_int\": 7 }", "{ \"foo_int\": 7 }");
+
+ JsonMerge("{ \"foo_uint\": 5 }", "{ \"foo_uint\": 7 }",
+ "{ \"foo_uint\": 7 }");
+
+ JsonMerge("{ \"foo_long\": 5 }", "{ \"foo_long\": 7 }",
+ "{ \"foo_long\": 7 }");
+
+ JsonMerge("{ \"foo_ulong\": 5 }", "{ \"foo_ulong\": 7 }",
+ "{ \"foo_ulong\": 7 }");
+
+ JsonMerge("{ \"foo_float\": 5.0 }", "{ \"foo_float\": 7.1 }",
+ "{ \"foo_float\": 7.1 }");
+
+ JsonMerge("{ \"foo_double\": 5.0 }", "{ \"foo_double\": 7.1 }",
+ "{ \"foo_double\": 7.1 }");
+}
+
+// Test arrays of simple types.
+TEST_F(FlatbufferMerge, Array) {
+ JsonMerge("{ \"foo_string\": \"baz\" }", "{ \"foo_string\": \"bar\" }",
+ "{ \"foo_string\": \"bar\" }");
+
+ JsonMerge("{ \"vector_foo_bool\": [ true, true, false ] }",
+ "{ \"vector_foo_bool\": [ false, true, true, false ] }",
+ "{ \"vector_foo_bool\": [ true, true, false, false, "
+ "true, true, false ] }");
+
+ JsonMerge("{ \"vector_foo_byte\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_byte\": [ -3, 1, 3, 2 ] }",
+ "{ \"vector_foo_byte\": [ 9, 7, 1, -3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_ubyte\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_ubyte\": [ 3, 1, 3, 2 ] }",
+ "{ \"vector_foo_ubyte\": [ 9, 7, 1, 3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_short\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_short\": [ -3, 1, 3, 2 ] }",
+ "{ \"vector_foo_short\": [ 9, 7, 1, -3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_ushort\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_ushort\": [ 3, 1, 3, 2 ] }",
+ "{ \"vector_foo_ushort\": [ 9, 7, 1, 3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_int\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_int\": [ -3, 1, 3, 2 ] }",
+ "{ \"vector_foo_int\": [ 9, 7, 1, -3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_uint\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_uint\": [ 3, 1, 3, 2 ] }",
+ "{ \"vector_foo_uint\": [ 9, 7, 1, 3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_long\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_long\": [ -3, 1, 3, 2 ] }",
+ "{ \"vector_foo_long\": [ 9, 7, 1, -3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_ulong\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_ulong\": [ 3, 1, 3, 2 ] }",
+ "{ \"vector_foo_ulong\": [ 9, 7, 1, 3, 1, 3, 2 ] }");
+
+ JsonMerge("{ \"vector_foo_float\": [ 9.0, 7.0, 1.0 ] }",
+ "{ \"vector_foo_float\": [ -3.0, 1.3, 3.0, 2.0 ] }",
+ "{ \"vector_foo_float\": [ 9.0, 7.0, 1.0, -3.0, 1.3, 3.0, 2.0 ] }");
+
+ JsonMerge(
+ "{ \"vector_foo_string\": [ \"9\", \"7\", \"1 \" ] }",
+ "{ \"vector_foo_string\": [ \"31\", \"32\" ] }",
+ "{ \"vector_foo_string\": [ \"9\", \"7\", \"1 \", \"31\", \"32\" ] }");
+}
+
+// Test nested messages, and arrays of nested messages.
+TEST_F(FlatbufferMerge, NestedStruct) {
+ JsonMerge(
+ "{ \"single_application\": { \"name\": \"woot\" } }",
+ "{ \"single_application\": { \"name\": \"wow\", \"priority\": 7 } }",
+ "{ \"single_application\": { \"name\": \"wow\", \"priority\": 7 } }");
+
+ JsonMerge(
+ "{ \"single_application\": { \"name\": \"wow\", \"priority\": 7 } }",
+ "{ \"single_application\": { \"name\": \"woot\" } }",
+ "{ \"single_application\": { \"name\": \"woot\", \"priority\": 7 } }");
+
+ JsonMerge("{ \"apps\": [ { \"name\": \"woot\" }, { \"name\": \"wow\" } ] }",
+ "{ \"apps\": [ { \"name\": \"woo2\" }, { \"name\": \"wo3\" } ] }",
+ "{ \"apps\": [ { \"name\": \"woot\" }, { \"name\": \"wow\" }, { "
+ "\"name\": \"woo2\" }, { \"name\": \"wo3\" } ] }");
+}
+
+// TODO(austin): enums
+// TODO(austin): unions
+// TODO(austin): struct
+
+} // namespace testing
+} // namespace aos
diff --git a/aos/flatbuffers.h b/aos/flatbuffers.h
new file mode 100644
index 0000000..6f6fc11
--- /dev/null
+++ b/aos/flatbuffers.h
@@ -0,0 +1,66 @@
+#ifndef AOS_FLATBUFFERS_H_
+#define AOS_FLATBUFFERS_H_
+
+#include "flatbuffers/flatbuffers.h"
+
+namespace aos {
+
+// TODO(austin): FlatbufferBase ? We can associate the type table with it, and
+// make it generic.
+
+// This object associates the message type with the memory storing the
+// flatbuffer. This only stores root tables.
+//
+// From a usage point of view, pointers to the data are very different than
+// pointers to the tables.
+template <typename T>
+class Flatbuffer {
+ public:
+ // Builds a Flatbuffer by taking ownership of the buffer.
+ Flatbuffer(flatbuffers::DetachedBuffer &&buffer)
+ : buffer_(::std::move(buffer)) {}
+
+ // Builds a flatbuffer by taking ownership of the buffer from the other
+ // flatbuffer.
+ Flatbuffer(Flatbuffer &&fb) : buffer_(::std::move(fb.buffer_)) {}
+ Flatbuffer &operator=(Flatbuffer &&fb) {
+ ::std::swap(buffer_, fb.buffer_);
+ return *this;
+ }
+
+ // Constructs an empty flatbuffer of type T.
+ static Flatbuffer<T> Empty() {
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(1);
+ const auto end = fbb.EndTable(fbb.StartTable());
+ fbb.Finish(flatbuffers::Offset<flatbuffers::Table>(end));
+ return Flatbuffer<T>(fbb.Release());
+ }
+
+ // Returns the MiniReflectTypeTable for T.
+ static const flatbuffers::TypeTable *MiniReflectTypeTable() {
+ return T::MiniReflectTypeTable();
+ }
+
+ // Returns a message from the buffer.
+ const T &message() const { return *flatbuffers::GetRoot<T>(buffer_.data()); }
+ // Returns a mutable message. It can be mutated via the flatbuffer rules.
+ T *mutable_message() {
+ return flatbuffers::GetMutableRoot<T>(buffer_.data());
+ }
+
+ // Returns references to the buffer, and the data.
+ const flatbuffers::DetachedBuffer &buffer() const { return buffer_; }
+ const uint8_t *data() const { return buffer_.data(); }
+
+ private:
+ flatbuffers::DetachedBuffer buffer_;
+};
+
+// TODO(austin): Need a fixed buffer version of Flatbuffer.
+// TODO(austin): Need a way to get our hands on the max size. Can start with
+// "large" for now.
+
+} // namespace aos
+
+#endif // AOS_FLATBUFFERS_H_
diff --git a/aos/json_to_flatbuffer.cc b/aos/json_to_flatbuffer.cc
index 85e2d7f..13a3a38 100644
--- a/aos/json_to_flatbuffer.cc
+++ b/aos/json_to_flatbuffer.cc
@@ -5,10 +5,10 @@
#include "absl/strings/string_view.h"
#include "aos/flatbuffer_utils.h"
-#include "aos/logging/logging.h"
#include "aos/json_tokenizer.h"
#include "flatbuffers/flatbuffers.h"
#include "flatbuffers/minireflect.h"
+#include "glog/logging.h"
// TODO(austin): Can we just do an Offset<void> ? It doesn't matter, so maybe
// just say that.
@@ -124,8 +124,8 @@
// Parses the json into a flatbuffer. Returns either an empty vector on
// error, or a vector with the flatbuffer data in it.
- ::std::vector<uint8_t> Parse(const absl::string_view data,
- const flatbuffers::TypeTable *typetable) {
+ flatbuffers::DetachedBuffer Parse(const absl::string_view data,
+ const flatbuffers::TypeTable *typetable) {
flatbuffers::uoffset_t end = 0;
bool result = DoParse(typetable, data, &end);
@@ -134,12 +134,10 @@
auto o = flatbuffers::Offset<flatbuffers::Table>(end);
fbb_.Finish(o);
- const uint8_t *buf = fbb_.GetBufferPointer();
- const int size = fbb_.GetSize();
- return ::std::vector<uint8_t>(buf, buf + size);
+ return fbb_.Release();
} else {
// Otherwise return an empty vector.
- return ::std::vector<uint8_t>();
+ return flatbuffers::DetachedBuffer();
}
}
@@ -333,14 +331,6 @@
if (!AddElement(field_index, double_value)) return false;
}
} break;
- // TODO(austin): Need to detect int vs float.
- /*
- asdf
- {
- const int field_index = stack_.back().field_index;
-
- } break;
- */
case Tokenizer::TokenType::kStringValue: // string value
{
const int field_index = stack_.back().field_index;
@@ -606,6 +596,7 @@
stack_.back().elements.emplace_back(
field_index, flatbuffers::Offset<flatbuffers::String>(
fbb_.EndVector(vector_elements_.size())));
+ vector_elements_.clear();
return true;
}
@@ -715,18 +706,37 @@
} // namespace
-::std::vector<uint8_t> JsonToFlatbuffer(
+flatbuffers::DetachedBuffer JsonToFlatbuffer(
const absl::string_view data, const flatbuffers::TypeTable *typetable) {
JsonParser p;
return p.Parse(data, typetable);
}
-::std::string FlatbufferToJson(const uint8_t *buffer,
- const ::flatbuffers::TypeTable *typetable,
- bool multi_line) {
+::std::string BufferFlatbufferToJson(const uint8_t *buffer,
+ const ::flatbuffers::TypeTable *typetable,
+ bool multi_line) {
+ // It is pretty common to get passed in a nullptr when a test fails. Rather
+ // than CHECK, return a more user friendly result.
+ if (buffer == nullptr) {
+ return "null";
+ }
+ return TableFlatbufferToJson(reinterpret_cast<const flatbuffers::Table *>(
+ flatbuffers::GetRoot<uint8_t>(buffer)),
+ typetable, multi_line);
+}
+
+::std::string TableFlatbufferToJson(const flatbuffers::Table *t,
+ const ::flatbuffers::TypeTable *typetable,
+ bool multi_line) {
+ // It is pretty common to get passed in a nullptr when a test fails. Rather
+ // than CHECK, return a more user friendly result.
+ if (t == nullptr) {
+ return "null";
+ }
::flatbuffers::ToStringVisitor tostring_visitor(
multi_line ? "\n" : " ", true, multi_line ? " " : "", multi_line);
- IterateFlatBuffer(buffer, typetable, &tostring_visitor);
+ flatbuffers::IterateObject(reinterpret_cast<const uint8_t *>(t), typetable,
+ &tostring_visitor);
return tostring_visitor.s;
}
diff --git a/aos/json_to_flatbuffer.fbs b/aos/json_to_flatbuffer.fbs
index 9743345..2e90a53 100644
--- a/aos/json_to_flatbuffer.fbs
+++ b/aos/json_to_flatbuffer.fbs
@@ -14,13 +14,14 @@
table Application {
name:string;
+ priority:int;
maps:[Map];
}
table Configuration {
locations:[Location] (id: 0);
maps:[Map] (id: 1);
- applications:[Application] (id: 2);
+ apps:[Application] (id: 2);
imports:[string] (id: 3);
// 8 bit: byte ubyte bool
@@ -68,6 +69,11 @@
// And a simple nested application.
single_application:Application (id: 28);
+
+ // TODO(austin): enum
+
+ // And nest this object to get some crazy coverage.
+ nested_config:Configuration (id: 29);
}
root_type Configuration;
diff --git a/aos/json_to_flatbuffer.h b/aos/json_to_flatbuffer.h
index 78d0703..183e6b2 100644
--- a/aos/json_to_flatbuffer.h
+++ b/aos/json_to_flatbuffer.h
@@ -5,20 +5,52 @@
#include <string>
#include "absl/strings/string_view.h"
+#include "aos/flatbuffers.h"
#include "flatbuffers/flatbuffers.h"
namespace aos {
// Parses the flatbuffer into the vector, or returns an empty vector.
-::std::vector<uint8_t> JsonToFlatbuffer(
+flatbuffers::DetachedBuffer JsonToFlatbuffer(
const absl::string_view data, const flatbuffers::TypeTable *typetable);
// Converts a flatbuffer into a Json string.
-//
// multi_line controls if the Json is written out on multiple lines or one.
-::std::string FlatbufferToJson(const uint8_t *buffer,
- const flatbuffers::TypeTable *typetable,
- bool multi_line = false);
+// The methods below are generally more useful than BufferFlatbufferToJson and
+// TableFlatbufferToJson.
+::std::string BufferFlatbufferToJson(const uint8_t *buffer,
+ const flatbuffers::TypeTable *typetable,
+ bool multi_line = false);
+
+::std::string TableFlatbufferToJson(const flatbuffers::Table *t,
+ const ::flatbuffers::TypeTable *typetable,
+ bool multi_line);
+
+// Converts a DetachedBuffer holding a flatbuffer to JSON.
+inline ::std::string FlatbufferToJson(const flatbuffers::DetachedBuffer &buffer,
+ const flatbuffers::TypeTable *typetable,
+ bool multi_line = false) {
+ return BufferFlatbufferToJson(buffer.data(), typetable, multi_line);
+}
+
+// Converts a Flatbuffer<T> holding a flatbuffer to JSON.
+template <typename T>
+inline ::std::string FlatbufferToJson(const Flatbuffer<T> &flatbuffer,
+ bool multi_line = false) {
+ return BufferFlatbufferToJson(
+ flatbuffer.data(), Flatbuffer<T>::MiniReflectTypeTable(), multi_line);
+}
+
+// Converts a flatbuffer::Table to JSON.
+template <typename T>
+typename std::enable_if<
+ std::is_base_of<flatbuffers::Table, T>::value,
+ std::string>::type inline FlatbufferToJson(const T *flatbuffer,
+ bool multi_line = false) {
+ return TableFlatbufferToJson(
+ reinterpret_cast<const flatbuffers::Table *>(flatbuffer),
+ Flatbuffer<T>::MiniReflectTypeTable(), multi_line);
+}
} // namespace aos
diff --git a/aos/json_to_flatbuffer_test.cc b/aos/json_to_flatbuffer_test.cc
index 4b0629e..a048e0a 100644
--- a/aos/json_to_flatbuffer_test.cc
+++ b/aos/json_to_flatbuffer_test.cc
@@ -16,15 +16,14 @@
bool JsonAndBack(const ::std::string in, const ::std::string out) {
printf("Testing: %s\n", in.c_str());
- const ::std::vector<uint8_t> fb =
+ const flatbuffers::DetachedBuffer fb =
JsonToFlatbuffer(in.data(), ConfigurationTypeTable());
if (fb.size() == 0) {
return false;
}
- const ::std::string back =
- FlatbufferToJson(fb.data(), ConfigurationTypeTable());
+ const ::std::string back = FlatbufferToJson(fb, ConfigurationTypeTable());
printf("Back to string: %s\n", back.c_str());
@@ -67,6 +66,7 @@
EXPECT_TRUE(JsonAndBack("{ \"foo_double\": 5.1 }"));
}
+
// Test what happens if you pass a field name that we don't know.
TEST_F(JsonToFlatbufferTest, InvalidFieldName) {
EXPECT_FALSE(JsonAndBack("{ \"foo\": 5 }"));
@@ -89,13 +89,12 @@
EXPECT_FALSE(JsonAndBack("{ \"foo_int\": 5, }", "{ \"foo_int\": 5 }"));
- EXPECT_FALSE(JsonAndBack(
- "{ \"applications\":\n[\n{\n\"name\": \"woot\"\n},\n{\n\"name\": "
- "\"wow\"\n} ,\n]\n}"));
-
EXPECT_FALSE(
- JsonAndBack("{ \"applications\": [ { \"name\": \"woot\" }, { \"name\": "
- "\"wow\" } ] , }"));
+ JsonAndBack("{ \"apps\":\n[\n{\n\"name\": \"woot\"\n},\n{\n\"name\": "
+ "\"wow\"\n} ,\n]\n}"));
+
+ EXPECT_FALSE(JsonAndBack(
+ "{ \"apps\": [ { \"name\": \"woot\" }, { \"name\": \"wow\" } ] , }"));
EXPECT_FALSE(
JsonAndBack("{ \"vector_foo_string\": [ \"bar\", \"baz\" ] , }"));
@@ -134,17 +133,35 @@
EXPECT_TRUE(
JsonAndBack("{ \"single_application\": { \"name\": \"woot\" } }"));
- EXPECT_TRUE(
- JsonAndBack("{ \"applications\": [ { \"name\": \"woot\" }, { \"name\": "
- "\"wow\" } ] }"));
+ EXPECT_TRUE(JsonAndBack(
+ "{ \"apps\": [ { \"name\": \"woot\" }, { \"name\": \"wow\" } ] }"));
}
// Test that we can parse an empty message.
TEST_F(JsonToFlatbufferTest, EmptyMessage) {
+ // Empty message works.
EXPECT_TRUE(JsonAndBack("{ }"));
}
+// Tests that comments get stripped.
+TEST_F(JsonToFlatbufferTest, Comments) {
+ EXPECT_TRUE(JsonAndBack("{ /* foo */ \"vector_foo_double\": [ 9, 7, 1 ] }",
+ "{ \"vector_foo_double\": [ 9.0, 7.0, 1.0 ] }"));
+}
+
+// Tests that multiple arrays get properly handled.
+TEST_F(JsonToFlatbufferTest, MultipleArrays) {
+ EXPECT_TRUE(
+ JsonAndBack("{ \"vector_foo_float\": [ 9, 7, 1 ], \"vector_foo_double\": "
+ "[ 9, 7, 1 ] }",
+ "{ \"vector_foo_float\": [ 9.0, 7.0, 1.0 ], "
+ "\"vector_foo_double\": [ 9.0, 7.0, 1.0 ] }"));
+}
+
// TODO(austin): Missmatched values.
+// TODO(austin): enums
+//
+// TODO(austin): unions?
} // namespace testing
} // namespace aos
diff --git a/aos/json_tokenizer.cc b/aos/json_tokenizer.cc
index 38ff4e3..78bf46e 100644
--- a/aos/json_tokenizer.cc
+++ b/aos/json_tokenizer.cc
@@ -13,6 +13,13 @@
} else if (Char() == '\n') {
ConsumeChar();
++linenumber_;
+ } else if (Consume("/*")) {
+ while (!Consume("*/")) {
+ if (Char() == '\n') {
+ ++linenumber_;
+ }
+ ConsumeChar();
+ }
} else {
// There is no fail. Once we are out of whitespace (including 0 of it),
// declare success.
diff --git a/third_party/flatbuffers/include/flatbuffers/base.h b/third_party/flatbuffers/include/flatbuffers/base.h
index d98da13..4eff0a6 100644
--- a/third_party/flatbuffers/include/flatbuffers/base.h
+++ b/third_party/flatbuffers/include/flatbuffers/base.h
@@ -212,6 +212,13 @@
typedef std::experimental::string_view string_view;
}
#define FLATBUFFERS_HAS_STRING_VIEW 1
+ // Check for absl::string_view (in c++14, compiler-dependent)
+ #elif __has_include("absl/strings/string_view.h")
+ #include "absl/strings/string_view.h"
+ namespace flatbuffers {
+ typedef absl::string_view string_view;
+ }
+ #define FLATBUFFERS_HAS_STRING_VIEW 1
#endif
#endif // __has_include
#endif // !FLATBUFFERS_HAS_STRING_VIEW
diff --git a/third_party/flatbuffers/include/flatbuffers/flatbuffers.h b/third_party/flatbuffers/include/flatbuffers/flatbuffers.h
index 1a250cd..f7b99ef 100644
--- a/third_party/flatbuffers/include/flatbuffers/flatbuffers.h
+++ b/third_party/flatbuffers/include/flatbuffers/flatbuffers.h
@@ -2341,6 +2341,9 @@
const uint8_t *GetVTable() const {
return data_ - ReadScalar<soffset_t>(data_);
}
+ uint8_t *GetMutableVTable() {
+ return data_ - ReadScalar<soffset_t>(data_);
+ }
// This gets the field offset for any of the functions below it, or 0
// if the field was not present.
@@ -2382,6 +2385,18 @@
return true;
}
+ void ClearField(voffset_t field) {
+ // The vtable offset is always at the start.
+ auto vtable = GetMutableVTable();
+ // The first element is the size of the vtable (fields + type id + itself).
+ auto vtsize = ReadScalar<voffset_t>(vtable);
+ // If the field we're accessing is outside the vtable, we're reading older
+ // data, so it's the same as if the offset was 0 (not present).
+ if (field < vtsize) {
+ WriteScalar<voffset_t>(reinterpret_cast<uint8_t *>(vtable + field), 0);
+ }
+ }
+
bool SetPointer(voffset_t field, const uint8_t *val) {
auto field_offset = GetOptionalFieldOffset(field);
if (!field_offset) return false;
diff --git a/third_party/flatbuffers/src/idl_gen_cpp.cpp b/third_party/flatbuffers/src/idl_gen_cpp.cpp
index b667ea4..0f9950d 100644
--- a/third_party/flatbuffers/src/idl_gen_cpp.cpp
+++ b/third_party/flatbuffers/src/idl_gen_cpp.cpp
@@ -1912,10 +1912,10 @@
}
if (parser_.opts.mutable_buffer) {
+ code_.SetValue("OFFSET_NAME", offset_str);
if (is_scalar) {
const auto type = GenTypeWire(field.value.type, "", false);
code_.SetValue("SET_FN", "SetField<" + type + ">");
- code_.SetValue("OFFSET_NAME", offset_str);
code_.SetValue("FIELD_TYPE", GenTypeBasic(field.value.type, true));
code_.SetValue("FIELD_VALUE",
GenUnderlyingCast(field, false, "_" + Name(field)));
@@ -1941,6 +1941,14 @@
code_ += " return {{FIELD_VALUE}};";
code_ += " }";
}
+
+ code_ += " void clear_{{FIELD_NAME}}() {";
+ code_ += " ClearField({{OFFSET_NAME}});";
+ code_ += " }";
+
+ code_ += " bool has_{{FIELD_NAME}}() const {";
+ code_ += " return CheckField({{OFFSET_NAME}});";
+ code_ += " }";
}
auto nested = field.attributes.Lookup("nested_flatbuffer");