Support structs in json_to_flatbuffer
Our JSON flatbuffer parsing code did not support structs properly.
Doing this enables easily defining lots of different layouts of
flatbuffers for more thoroughly testing flatbuffer-related changes (as
well as just making the JSON parsing more complete).
Change-Id: Ibfc7a149f9c8b314f6bbba70b4ca6b5857a8728b
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/aos/json_to_flatbuffer.cc b/aos/json_to_flatbuffer.cc
index 023deea..c16094c 100644
--- a/aos/json_to_flatbuffer.cc
+++ b/aos/json_to_flatbuffer.cc
@@ -23,10 +23,10 @@
// Class to hold one of the 3 json types for an array.
struct Element {
// The type.
- enum class ElementType { INT, DOUBLE, OFFSET };
+ enum class ElementType { INT, DOUBLE, OFFSET, STRUCT };
// Constructs an Element holding an integer.
- Element(int64_t new_int_element)
+ Element(absl::int128 new_int_element)
: int_element(new_int_element), type(ElementType::INT) {}
// Constructs an Element holding an double.
Element(double new_double_element)
@@ -34,13 +34,24 @@
// Constructs an Element holding an Offset.
Element(flatbuffers::Offset<flatbuffers::String> new_offset_element)
: offset_element(new_offset_element), type(ElementType::OFFSET) {}
+ // Constructs an Element holding a struct.
+ Element(std::vector<uint8_t> struct_data)
+ : /*initialize the union member to keep the compiler happy*/ int_element(
+ 0),
+ struct_data(std::move(struct_data)),
+ type(ElementType::STRUCT) {}
// Union for the various datatypes.
union {
- int64_t int_element;
+ absl::int128 int_element;
double double_element;
flatbuffers::Offset<flatbuffers::String> offset_element;
};
+ // Because we can't know the maximum size of any potential structs at
+ // compile-time, we will use a vector to store the vector data inline.
+ // If you were to do a reinterpret_cast<StructType*>(struct_data.data()) then
+ // you would have an instance of the struct in question.
+ std::vector<uint8_t> struct_data;
// And an enum signaling which one is in use.
ElementType type;
@@ -48,13 +59,15 @@
// Structure to represent a field element.
struct FieldElement {
- FieldElement(int new_field_index, int64_t int_element)
+ FieldElement(int new_field_index, absl::int128 int_element)
: element(int_element), field_index(new_field_index) {}
FieldElement(int new_field_index, double double_element)
: element(double_element), field_index(new_field_index) {}
FieldElement(int new_field_index,
flatbuffers::Offset<flatbuffers::String> offset_element)
: element(offset_element), field_index(new_field_index) {}
+ FieldElement(int new_field_index, const Element &element)
+ : element(element), field_index(new_field_index) {}
// Data to write.
Element element;
@@ -68,29 +81,179 @@
bool AddSingleElement(FlatbufferType type, const FieldElement &field_element,
::std::vector<bool> *fields_in_use,
flatbuffers::FlatBufferBuilder *fbb);
-bool AddSingleElement(FlatbufferType type, int field_index, int64_t int_value,
+bool AddSingleElement(FlatbufferType type, int field_index,
+ absl::int128 int_value,
flatbuffers::FlatBufferBuilder *fbb);
bool AddSingleElement(FlatbufferType type, int field_index, double double_value,
flatbuffers::FlatBufferBuilder *fbb);
bool AddSingleElement(FlatbufferType type, int field_index,
flatbuffers::Offset<flatbuffers::String> offset_element,
flatbuffers::FlatBufferBuilder *fbb);
+bool AddSingleElement(FlatbufferType type, int field_index,
+ const std::vector<uint8_t> &struct_data,
+ flatbuffers::FlatBufferBuilder *fbb);
+
+template <typename T, typename U>
+void SetMemory(U value, uint8_t *destination) {
+ // destination may be poorly aligned. As such, we should not simply do
+ // *reinterpret_cast<T*>(destination) = value directly.
+ const T casted = static_cast<T>(value);
+ memcpy(destination, &casted, sizeof(T));
+}
+
+bool SetStructElement(FlatbufferType type, int field_index, absl::int128 value,
+ uint8_t *destination) {
+ const flatbuffers::ElementaryType elementary_type =
+ type.FieldElementaryType(field_index);
+ switch (elementary_type) {
+ case flatbuffers::ET_CHAR:
+ SetMemory<int8_t>(value, destination);
+ break;
+ case flatbuffers::ET_UCHAR:
+ SetMemory<uint8_t>(value, destination);
+ break;
+ case flatbuffers::ET_SHORT:
+ SetMemory<int16_t>(value, destination);
+ break;
+ case flatbuffers::ET_USHORT:
+ SetMemory<uint16_t>(value, destination);
+ break;
+ case flatbuffers::ET_INT:
+ SetMemory<int32_t>(value, destination);
+ break;
+ case flatbuffers::ET_UINT:
+ SetMemory<uint32_t>(value, destination);
+ break;
+ case flatbuffers::ET_LONG:
+ SetMemory<int64_t>(value, destination);
+ break;
+ case flatbuffers::ET_ULONG:
+ SetMemory<uint64_t>(value, destination);
+ break;
+ case flatbuffers::ET_BOOL:
+ SetMemory<bool>(value, destination);
+ break;
+ case flatbuffers::ET_FLOAT:
+ SetMemory<float>(value, destination);
+ break;
+ case flatbuffers::ET_DOUBLE:
+ SetMemory<double>(value, destination);
+ break;
+ case flatbuffers::ET_STRING:
+ case flatbuffers::ET_UTYPE:
+ case flatbuffers::ET_SEQUENCE: {
+ const std::string_view name = type.FieldName(field_index);
+ fprintf(stderr,
+ "Mismatched type for field '%.*s'. Got: integer, expected %s\n",
+ static_cast<int>(name.size()), name.data(),
+ ElementaryTypeName(elementary_type));
+ return false;
+ }
+ }
+ return true;
+}
+
+bool SetStructElement(FlatbufferType type, int field_index, double value,
+ uint8_t *destination) {
+ const flatbuffers::ElementaryType elementary_type =
+ type.FieldElementaryType(field_index);
+ switch (elementary_type) {
+ case flatbuffers::ET_FLOAT:
+ SetMemory<float>(value, destination);
+ break;
+ case flatbuffers::ET_DOUBLE:
+ SetMemory<double>(value, destination);
+ break;
+ case flatbuffers::ET_CHAR:
+ case flatbuffers::ET_UCHAR:
+ case flatbuffers::ET_SHORT:
+ case flatbuffers::ET_USHORT:
+ case flatbuffers::ET_INT:
+ case flatbuffers::ET_UINT:
+ case flatbuffers::ET_LONG:
+ case flatbuffers::ET_ULONG:
+ case flatbuffers::ET_BOOL:
+ case flatbuffers::ET_STRING:
+ case flatbuffers::ET_UTYPE:
+ case flatbuffers::ET_SEQUENCE: {
+ const std::string_view name = type.FieldName(field_index);
+ fprintf(stderr,
+ "Mismatched type for field '%.*s'. Got: integer, expected %s\n",
+ static_cast<int>(name.size()), name.data(),
+ ElementaryTypeName(elementary_type));
+ return false;
+ }
+ }
+ return true;
+}
// Writes an array of FieldElement (with the definition in "type") to the
// builder. Returns the offset of the resulting table.
-flatbuffers::uoffset_t WriteTable(FlatbufferType type,
- const ::std::vector<FieldElement> &elements,
- flatbuffers::FlatBufferBuilder *fbb) {
- // End of a nested struct! Add it.
- const flatbuffers::uoffset_t start = fbb->StartTable();
+std::optional<Element> WriteObject(FlatbufferType type,
+ const ::std::vector<FieldElement> &elements,
+ flatbuffers::FlatBufferBuilder *fbb) {
+ // End of a nested object! Add it.
+ if (type.IsTable()) {
+ const flatbuffers::uoffset_t start = fbb->StartTable();
- ::std::vector<bool> fields_in_use(type.NumberFields(), false);
+ ::std::vector<bool> fields_in_use(type.NumberFields(), false);
- for (const FieldElement &field_element : elements) {
- AddSingleElement(type, field_element, &fields_in_use, fbb);
+ for (const FieldElement &field_element : elements) {
+ AddSingleElement(type, field_element, &fields_in_use, fbb);
+ }
+
+ return Element{
+ flatbuffers::Offset<flatbuffers::String>{fbb->EndTable(start)}};
+ } else if (type.IsStruct()) {
+ // In order to write an inline struct, we need to fill out each field at the
+ // correct position inline in memory. In order to do this, we retrieve the
+ // offset/size of each field, and directly populate that memory with the
+ // relevant value.
+ std::vector<uint8_t> buffer(type.InlineSize(), 0);
+ for (size_t field_index = 0;
+ field_index < static_cast<size_t>(type.NumberFields());
+ ++field_index) {
+ auto it = std::find_if(elements.begin(), elements.end(),
+ [field_index](const FieldElement &field) {
+ return field.field_index ==
+ static_cast<int>(field_index);
+ });
+ if (it == elements.end()) {
+ fprintf(stderr,
+ "All fields must be specified for struct types (field %s "
+ "missing).\n",
+ type.FieldName(field_index).data());
+ return std::nullopt;
+ }
+
+ uint8_t *field_data = buffer.data() + type.StructFieldOffset(field_index);
+ const size_t field_size = type.FieldInlineSize(field_index);
+ switch (it->element.type) {
+ case Element::ElementType::INT:
+ if (!SetStructElement(type, field_index, it->element.int_element,
+ field_data)) {
+ return std::nullopt;
+ }
+ break;
+ case Element::ElementType::DOUBLE:
+ if (!SetStructElement(type, field_index, it->element.double_element,
+ field_data)) {
+ return std::nullopt;
+ }
+ break;
+ case Element::ElementType::STRUCT:
+ CHECK_EQ(field_size, it->element.struct_data.size());
+ memcpy(field_data, it->element.struct_data.data(), field_size);
+ break;
+ case Element::ElementType::OFFSET:
+ LOG(FATAL)
+ << "This should be unreachable; structs cannot contain offsets.";
+ break;
+ }
+ }
+ return Element{buffer};
}
-
- return fbb->EndTable(start);
+ LOG(FATAL) << "Unimplemented.";
}
// Class to parse JSON into a flatbuffer.
@@ -135,7 +298,7 @@
// Adds *_value for the provided field. If we are in a vector, queues the
// data up in vector_elements. Returns true on success.
- bool AddElement(int field_index, int64_t int_value);
+ bool AddElement(int field_index, absl::int128 int_value);
bool AddElement(int field_index, double double_value);
bool AddElement(int field_index, const ::std::string &data);
@@ -144,12 +307,13 @@
// Pushes an element as part of a vector. Returns true on success.
bool PushElement(flatbuffers::ElementaryType elementary_type,
- int64_t int_value);
+ absl::int128 int_value);
bool PushElement(flatbuffers::ElementaryType elementary_type,
double double_value);
bool PushElement(flatbuffers::ElementaryType elementary_type,
flatbuffers::Offset<flatbuffers::String> offset_value);
-
+ bool PushElement(const FlatbufferType &type,
+ const std::vector<uint8_t> &struct_data);
flatbuffers::FlatBufferBuilder *fbb_;
// This holds the state information that is needed as you recurse into
@@ -170,8 +334,8 @@
// For scalar types (not strings, and not nested tables), the vector ends
// up being implemented as a start and end, and a block of data. So we
// can't just push offsets in as we go. We either need to reproduce the
- // logic inside flatbuffers, or build up vectors of the data. Vectors will
- // be a bit of extra stack space, but whatever.
+ // logic inside flatbuffers, or build up vectors of the data. Vectors
+ // will be a bit of extra stack space, but whatever.
//
// Strings and nested structures are vectors of offsets.
// into the vector. Once you get to the end, you build up a vector and
@@ -238,17 +402,23 @@
fprintf(stderr, "Empty stack\n");
return false;
} else {
- // End of a nested struct! Add it.
- const flatbuffers::uoffset_t end =
- WriteTable(stack_.back().type, stack_.back().elements, fbb_);
+ // End of a nested object! Add it.
+ std::optional<Element> object =
+ WriteObject(stack_.back().type, stack_.back().elements, fbb_);
+ if (!object.has_value()) {
+ return false;
+ }
// We now want to talk about the parent structure. Pop the child.
stack_.pop_back();
if (stack_.size() == 0) {
+ CHECK_EQ(static_cast<int>(object->type),
+ static_cast<int>(Element::ElementType::OFFSET))
+ << ": JSON parsing only supports parsing flatbuffer tables.";
// Instead of queueing it up in the stack, return it through the
// passed in variable.
- *table_end = end;
+ *table_end = object->offset_element.o;
} else {
// And now we can add it.
const int field_index = stack_.back().field_index;
@@ -256,10 +426,10 @@
// Do the right thing if we are in a vector.
if (in_vector()) {
stack_.back().vector_elements.emplace_back(
- flatbuffers::Offset<flatbuffers::String>(end));
+ std::move(object.value()));
} else {
- stack_.back().elements.emplace_back(
- field_index, flatbuffers::Offset<flatbuffers::String>(end));
+ stack_.back().elements.emplace_back(field_index,
+ std::move(object.value()));
}
}
}
@@ -294,7 +464,7 @@
case Tokenizer::TokenType::kNumberValue: {
bool is_int = true;
double double_value;
- long long int_value;
+ absl::int128 int_value;
if (token == Tokenizer::TokenType::kTrueValue) {
int_value = 1;
} else if (token == Tokenizer::TokenType::kFalseValue) {
@@ -313,7 +483,7 @@
if (is_int) {
// No need to get too stressed about bool vs int. Convert them all.
- int64_t val = int_value;
+ absl::int128 val = int_value;
if (!AddElement(field_index, val)) return false;
} else {
if (!AddElement(field_index, double_value)) return false;
@@ -342,7 +512,7 @@
return false;
}
-bool JsonParser::AddElement(int field_index, int64_t int_value) {
+bool JsonParser::AddElement(int field_index, absl::int128 int_value) {
if (stack_.back().type.FieldIsRepeating(field_index) != in_vector()) {
fprintf(stderr, "Type and json disagree on if we are in a vector or not\n");
return false;
@@ -393,7 +563,7 @@
const FlatbufferType enum_type = type.FieldType(field_index);
CHECK(enum_type.IsEnum());
- const std::optional<int64_t> int_value = enum_type.EnumValue(data);
+ const std::optional<absl::int128> int_value = enum_type.EnumValue(data);
if (!int_value) {
const std::string_view name = type.FieldName(field_index);
@@ -448,11 +618,15 @@
case Element::ElementType::OFFSET:
return AddSingleElement(type, field_element.field_index,
field_element.element.offset_element, fbb);
+ case Element::ElementType::STRUCT:
+ return AddSingleElement(type, field_element.field_index,
+ field_element.element.struct_data, fbb);
}
return false;
}
-bool AddSingleElement(FlatbufferType type, int field_index, int64_t int_value,
+bool AddSingleElement(FlatbufferType type, int field_index,
+ absl::int128 int_value,
flatbuffers::FlatBufferBuilder *fbb
) {
@@ -463,37 +637,39 @@
type.FieldElementaryType(field_index);
switch (elementary_type) {
case flatbuffers::ET_BOOL:
- fbb->AddElement<bool>(field_offset, int_value);
+ fbb->AddElement<bool>(field_offset, static_cast<bool>(int_value));
return true;
case flatbuffers::ET_CHAR:
- fbb->AddElement<int8_t>(field_offset, int_value);
+ fbb->AddElement<int8_t>(field_offset, static_cast<int8_t>(int_value));
return true;
case flatbuffers::ET_UCHAR:
- fbb->AddElement<uint8_t>(field_offset, int_value);
+ fbb->AddElement<uint8_t>(field_offset, static_cast<uint8_t>(int_value));
return true;
case flatbuffers::ET_SHORT:
- fbb->AddElement<int16_t>(field_offset, int_value);
+ fbb->AddElement<int16_t>(field_offset, static_cast<int16_t>(int_value));
return true;
case flatbuffers::ET_USHORT:
- fbb->AddElement<uint16_t>(field_offset, int_value);
+ fbb->AddElement<uint16_t>(field_offset, static_cast<uint16_t>(int_value));
return true;
case flatbuffers::ET_INT:
- fbb->AddElement<int32_t>(field_offset, int_value);
+ fbb->AddElement<int32_t>(field_offset, static_cast<int32_t>(int_value));
return true;
case flatbuffers::ET_UINT:
- fbb->AddElement<uint32_t>(field_offset, int_value);
+ fbb->AddElement<uint32_t>(field_offset, static_cast<uint32_t>(int_value));
return true;
case flatbuffers::ET_LONG:
- fbb->AddElement<int64_t>(field_offset, int_value);
+ fbb->AddElement<int64_t>(field_offset, static_cast<int64_t>(int_value));
return true;
case flatbuffers::ET_ULONG:
- fbb->AddElement<uint64_t>(field_offset, int_value);
+ fbb->AddElement<uint64_t>(field_offset, static_cast<uint64_t>(int_value));
return true;
+ // The floating point cases occur when someone specifies an integer in the
+ // JSON for a double field.
case flatbuffers::ET_FLOAT:
- fbb->AddElement<float>(field_offset, int_value);
+ fbb->AddElement<float>(field_offset, static_cast<float>(int_value));
return true;
case flatbuffers::ET_DOUBLE:
- fbb->AddElement<double>(field_offset, int_value);
+ fbb->AddElement<double>(field_offset, static_cast<double>(int_value));
return true;
case flatbuffers::ET_STRING:
case flatbuffers::ET_UTYPE:
@@ -545,6 +721,7 @@
}
return false;
}
+
bool AddSingleElement(FlatbufferType type, int field_index,
flatbuffers::Offset<flatbuffers::String> offset_element,
flatbuffers::FlatBufferBuilder *fbb) {
@@ -587,11 +764,27 @@
return false;
}
+bool AddSingleElement(FlatbufferType type, int field_index,
+ const std::vector<uint8_t> &data,
+ flatbuffers::FlatBufferBuilder *fbb) {
+ // Structs are always inline.
+ // We have to do somewhat manual serialization to get the struct into place,
+ // since the regular FlatBufferBuilder assumes that you will know the type of
+ // the struct that you are constructing at compile time.
+ fbb->Align(type.FieldType(field_index).Alignment());
+ fbb->PushBytes(data.data(), data.size());
+ fbb->AddStructOffset(flatbuffers::FieldIndexToOffset(
+ static_cast<flatbuffers::voffset_t>(field_index)),
+ fbb->GetSize());
+ return true;
+}
+
bool JsonParser::FinishVector(int field_index) {
// Vectors have a start (unfortunately which needs to know the size)
const size_t inline_size = stack_.back().type.FieldInlineSize(field_index);
+ const size_t alignment = stack_.back().type.FieldInlineAlignment(field_index);
fbb_->StartVector(stack_.back().vector_elements.size(), inline_size,
- /*align=*/inline_size);
+ /*align=*/alignment);
const flatbuffers::ElementaryType elementary_type =
stack_.back().type.FieldElementaryType(field_index);
@@ -609,6 +802,11 @@
case Element::ElementType::OFFSET:
if (!PushElement(elementary_type, element.offset_element)) return false;
break;
+ case Element::ElementType::STRUCT:
+ if (!PushElement(stack_.back().type.FieldType(field_index),
+ element.struct_data))
+ return false;
+ break;
}
}
@@ -621,40 +819,40 @@
}
bool JsonParser::PushElement(flatbuffers::ElementaryType elementary_type,
- int64_t int_value) {
+ absl::int128 int_value) {
switch (elementary_type) {
case flatbuffers::ET_BOOL:
- fbb_->PushElement<bool>(int_value);
+ fbb_->PushElement<bool>(static_cast<bool>(int_value));
return true;
case flatbuffers::ET_CHAR:
- fbb_->PushElement<int8_t>(int_value);
+ fbb_->PushElement<int8_t>(static_cast<int8_t>(int_value));
return true;
case flatbuffers::ET_UCHAR:
- fbb_->PushElement<uint8_t>(int_value);
+ fbb_->PushElement<uint8_t>(static_cast<uint8_t>(int_value));
return true;
case flatbuffers::ET_SHORT:
- fbb_->PushElement<int16_t>(int_value);
+ fbb_->PushElement<int16_t>(static_cast<int16_t>(int_value));
return true;
case flatbuffers::ET_USHORT:
- fbb_->PushElement<uint16_t>(int_value);
+ fbb_->PushElement<uint16_t>(static_cast<uint16_t>(int_value));
return true;
case flatbuffers::ET_INT:
- fbb_->PushElement<int32_t>(int_value);
+ fbb_->PushElement<int32_t>(static_cast<int32_t>(int_value));
return true;
case flatbuffers::ET_UINT:
- fbb_->PushElement<uint32_t>(int_value);
+ fbb_->PushElement<uint32_t>(static_cast<uint32_t>(int_value));
return true;
case flatbuffers::ET_LONG:
- fbb_->PushElement<int64_t>(int_value);
+ fbb_->PushElement<int64_t>(static_cast<int64_t>(int_value));
return true;
case flatbuffers::ET_ULONG:
- fbb_->PushElement<uint64_t>(int_value);
+ fbb_->PushElement<uint64_t>(static_cast<uint64_t>(int_value));
return true;
case flatbuffers::ET_FLOAT:
- fbb_->PushElement<float>(int_value);
+ fbb_->PushElement<float>(static_cast<float>(int_value));
return true;
case flatbuffers::ET_DOUBLE:
- fbb_->PushElement<double>(int_value);
+ fbb_->PushElement<double>(static_cast<double>(int_value));
return true;
case flatbuffers::ET_STRING:
case flatbuffers::ET_UTYPE:
@@ -698,6 +896,17 @@
return false;
}
+bool JsonParser::PushElement(const FlatbufferType &type,
+ const std::vector<uint8_t> &struct_data) {
+ // To add a struct to a vector, we just need to get the relevant bytes pushed
+ // straight into the builder. The FlatBufferBuilder normally expects that you
+ // will know the type of your struct at compile-time, so doesn't have a
+ // first-class way to do this.
+ fbb_->Align(type.Alignment());
+ fbb_->PushBytes(struct_data.data(), struct_data.size());
+ return true;
+}
+
bool JsonParser::PushElement(
flatbuffers::ElementaryType elementary_type,
flatbuffers::Offset<flatbuffers::String> offset_value) {