Support structs in json_to_flatbuffer
Our JSON flatbuffer parsing code did not support structs properly.
Doing this enables easily defining lots of different layouts of
flatbuffers for more thoroughly testing flatbuffer-related changes (as
well as just making the JSON parsing more complete).
Change-Id: Ibfc7a149f9c8b314f6bbba70b4ca6b5857a8728b
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/aos/flatbuffer_utils.cc b/aos/flatbuffer_utils.cc
index 3429fe4..5abe676 100644
--- a/aos/flatbuffer_utils.cc
+++ b/aos/flatbuffer_utils.cc
@@ -19,6 +19,26 @@
LOG(FATAL) << "Unimplemented";
}
+bool FlatbufferType::IsTable() const {
+ if (type_table_) {
+ return type_table_->st == flatbuffers::ST_TABLE;
+ }
+ if (object_) {
+ return !object_->is_struct();
+ }
+ LOG(FATAL) << "Unimplemented";
+}
+
+bool FlatbufferType::IsStruct() const {
+ if (type_table_) {
+ return type_table_->st == flatbuffers::ST_STRUCT;
+ }
+ if (object_) {
+ return object_->is_struct();
+ }
+ LOG(FATAL) << "Unimplemented";
+}
+
bool FlatbufferType::IsEnum() const {
if (type_table_) {
return type_table_->st == flatbuffers::ST_ENUM;
@@ -256,10 +276,64 @@
} // namespace
+size_t FlatbufferType::InlineSize() const {
+ DCHECK(IsSequence());
+ if (type_table_) {
+ return flatbuffers::InlineSize(flatbuffers::ElementaryType::ET_SEQUENCE,
+ type_table_);
+ }
+ if (object_) {
+ return object_->is_struct() ? object_->bytesize() : /*offset size*/ 4u;
+ }
+ if (enum_) {
+ return BaseTypeInlineSize(enum_->underlying_type()->base_type());
+ }
+ LOG(FATAL) << "Unimplemented";
+}
+
+// Returns the required alignment for this type.
+size_t FlatbufferType::Alignment() const {
+ if (type_table_) {
+ // Attempt to derive alignment as max alignment of the members.
+ size_t alignment = 1u;
+ for (size_t field_index = 0;
+ field_index < static_cast<size_t>(NumberFields()); ++field_index) {
+ alignment = std::max(alignment, FieldInlineAlignment(field_index));
+ }
+ return alignment;
+ }
+ if (object_) {
+ return object_->minalign();
+ }
+ // We don't do a great job of supporting unions in general, and as of this
+ // writing did not try to look up what the alignment rules for unions were.
+ LOG(FATAL) << "Unimplemented";
+}
+
+size_t FlatbufferType::FieldInlineAlignment(size_t field_index) const {
+ if (FieldIsSequence(field_index) && FieldType(field_index).IsStruct()) {
+ return FieldType(field_index).Alignment();
+ }
+ return FieldInlineSize(field_index);
+}
+
+size_t FlatbufferType::StructFieldOffset(int index) const {
+ DCHECK(IsStruct());
+ if (type_table_) {
+ return type_table_->values[index];
+ }
+ if (object_) {
+ return ReflectionObjectField(index)->offset();
+ }
+ LOG(FATAL) << "Unimplemented";
+}
+
size_t FlatbufferType::FieldInlineSize(int index) const {
DCHECK(IsSequence());
if (type_table_) {
- return flatbuffers::InlineSize(FieldElementaryType(index), type_table_);
+ return flatbuffers::InlineSize(
+ FieldElementaryType(index),
+ FieldIsSequence(index) ? FieldType(index).type_table_ : nullptr);
}
if (object_ || enum_) {
const reflection::Type *const type = ReflectionType(index);
diff --git a/aos/flatbuffer_utils.h b/aos/flatbuffer_utils.h
index 0287b6d..aa22e68 100644
--- a/aos/flatbuffer_utils.h
+++ b/aos/flatbuffer_utils.h
@@ -43,6 +43,12 @@
// Returns whether this type is an enum.
bool IsEnum() const;
+ // Returns whether this type is a struct.
+ bool IsStruct() const;
+
+ // Returns whether this type is a table.
+ bool IsTable() const;
+
// Returns whether the given field is a sequence (table, struct, or union).
//
// Only valid for sequences (tables, structs, or unions).
@@ -84,6 +90,7 @@
//
// Only valid for sequences (tables, structs, or unions).
size_t FieldInlineSize(int index) const;
+ size_t InlineSize() const;
// Returns the total number of fields.
//
@@ -95,6 +102,16 @@
// Only valid for sequences (tables, structs, or unions).
FlatbufferType FieldType(int index) const;
+ // Returns the offset of the specified field within the struct.
+ //
+ // Only valid for structs.
+ size_t StructFieldOffset(int index) const;
+
+ // Returns the required alignment for this type.
+ size_t Alignment() const;
+ // The required alignment of the inline data for the specified field.
+ size_t FieldInlineAlignment(size_t field_index) const;
+
private:
explicit FlatbufferType(const reflection::Schema *schema,
const reflection::Object *object)
diff --git a/aos/json_to_flatbuffer.cc b/aos/json_to_flatbuffer.cc
index 023deea..c16094c 100644
--- a/aos/json_to_flatbuffer.cc
+++ b/aos/json_to_flatbuffer.cc
@@ -23,10 +23,10 @@
// Class to hold one of the 3 json types for an array.
struct Element {
// The type.
- enum class ElementType { INT, DOUBLE, OFFSET };
+ enum class ElementType { INT, DOUBLE, OFFSET, STRUCT };
// Constructs an Element holding an integer.
- Element(int64_t new_int_element)
+ Element(absl::int128 new_int_element)
: int_element(new_int_element), type(ElementType::INT) {}
// Constructs an Element holding an double.
Element(double new_double_element)
@@ -34,13 +34,24 @@
// Constructs an Element holding an Offset.
Element(flatbuffers::Offset<flatbuffers::String> new_offset_element)
: offset_element(new_offset_element), type(ElementType::OFFSET) {}
+ // Constructs an Element holding a struct.
+ Element(std::vector<uint8_t> struct_data)
+ : /*initialize the union member to keep the compiler happy*/ int_element(
+ 0),
+ struct_data(std::move(struct_data)),
+ type(ElementType::STRUCT) {}
// Union for the various datatypes.
union {
- int64_t int_element;
+ absl::int128 int_element;
double double_element;
flatbuffers::Offset<flatbuffers::String> offset_element;
};
+ // Because we can't know the maximum size of any potential structs at
+ // compile-time, we will use a vector to store the vector data inline.
+ // If you were to do a reinterpret_cast<StructType*>(struct_data.data()) then
+ // you would have an instance of the struct in question.
+ std::vector<uint8_t> struct_data;
// And an enum signaling which one is in use.
ElementType type;
@@ -48,13 +59,15 @@
// Structure to represent a field element.
struct FieldElement {
- FieldElement(int new_field_index, int64_t int_element)
+ FieldElement(int new_field_index, absl::int128 int_element)
: element(int_element), field_index(new_field_index) {}
FieldElement(int new_field_index, double double_element)
: element(double_element), field_index(new_field_index) {}
FieldElement(int new_field_index,
flatbuffers::Offset<flatbuffers::String> offset_element)
: element(offset_element), field_index(new_field_index) {}
+ FieldElement(int new_field_index, const Element &element)
+ : element(element), field_index(new_field_index) {}
// Data to write.
Element element;
@@ -68,29 +81,179 @@
bool AddSingleElement(FlatbufferType type, const FieldElement &field_element,
::std::vector<bool> *fields_in_use,
flatbuffers::FlatBufferBuilder *fbb);
-bool AddSingleElement(FlatbufferType type, int field_index, int64_t int_value,
+bool AddSingleElement(FlatbufferType type, int field_index,
+ absl::int128 int_value,
flatbuffers::FlatBufferBuilder *fbb);
bool AddSingleElement(FlatbufferType type, int field_index, double double_value,
flatbuffers::FlatBufferBuilder *fbb);
bool AddSingleElement(FlatbufferType type, int field_index,
flatbuffers::Offset<flatbuffers::String> offset_element,
flatbuffers::FlatBufferBuilder *fbb);
+bool AddSingleElement(FlatbufferType type, int field_index,
+ const std::vector<uint8_t> &struct_data,
+ flatbuffers::FlatBufferBuilder *fbb);
+
+template <typename T, typename U>
+void SetMemory(U value, uint8_t *destination) {
+ // destination may be poorly aligned. As such, we should not simply do
+ // *reinterpret_cast<T*>(destination) = value directly.
+ const T casted = static_cast<T>(value);
+ memcpy(destination, &casted, sizeof(T));
+}
+
+bool SetStructElement(FlatbufferType type, int field_index, absl::int128 value,
+ uint8_t *destination) {
+ const flatbuffers::ElementaryType elementary_type =
+ type.FieldElementaryType(field_index);
+ switch (elementary_type) {
+ case flatbuffers::ET_CHAR:
+ SetMemory<int8_t>(value, destination);
+ break;
+ case flatbuffers::ET_UCHAR:
+ SetMemory<uint8_t>(value, destination);
+ break;
+ case flatbuffers::ET_SHORT:
+ SetMemory<int16_t>(value, destination);
+ break;
+ case flatbuffers::ET_USHORT:
+ SetMemory<uint16_t>(value, destination);
+ break;
+ case flatbuffers::ET_INT:
+ SetMemory<int32_t>(value, destination);
+ break;
+ case flatbuffers::ET_UINT:
+ SetMemory<uint32_t>(value, destination);
+ break;
+ case flatbuffers::ET_LONG:
+ SetMemory<int64_t>(value, destination);
+ break;
+ case flatbuffers::ET_ULONG:
+ SetMemory<uint64_t>(value, destination);
+ break;
+ case flatbuffers::ET_BOOL:
+ SetMemory<bool>(value, destination);
+ break;
+ case flatbuffers::ET_FLOAT:
+ SetMemory<float>(value, destination);
+ break;
+ case flatbuffers::ET_DOUBLE:
+ SetMemory<double>(value, destination);
+ break;
+ case flatbuffers::ET_STRING:
+ case flatbuffers::ET_UTYPE:
+ case flatbuffers::ET_SEQUENCE: {
+ const std::string_view name = type.FieldName(field_index);
+ fprintf(stderr,
+ "Mismatched type for field '%.*s'. Got: integer, expected %s\n",
+ static_cast<int>(name.size()), name.data(),
+ ElementaryTypeName(elementary_type));
+ return false;
+ }
+ }
+ return true;
+}
+
+bool SetStructElement(FlatbufferType type, int field_index, double value,
+ uint8_t *destination) {
+ const flatbuffers::ElementaryType elementary_type =
+ type.FieldElementaryType(field_index);
+ switch (elementary_type) {
+ case flatbuffers::ET_FLOAT:
+ SetMemory<float>(value, destination);
+ break;
+ case flatbuffers::ET_DOUBLE:
+ SetMemory<double>(value, destination);
+ break;
+ case flatbuffers::ET_CHAR:
+ case flatbuffers::ET_UCHAR:
+ case flatbuffers::ET_SHORT:
+ case flatbuffers::ET_USHORT:
+ case flatbuffers::ET_INT:
+ case flatbuffers::ET_UINT:
+ case flatbuffers::ET_LONG:
+ case flatbuffers::ET_ULONG:
+ case flatbuffers::ET_BOOL:
+ case flatbuffers::ET_STRING:
+ case flatbuffers::ET_UTYPE:
+ case flatbuffers::ET_SEQUENCE: {
+ const std::string_view name = type.FieldName(field_index);
+ fprintf(stderr,
+ "Mismatched type for field '%.*s'. Got: integer, expected %s\n",
+ static_cast<int>(name.size()), name.data(),
+ ElementaryTypeName(elementary_type));
+ return false;
+ }
+ }
+ return true;
+}
// Writes an array of FieldElement (with the definition in "type") to the
// builder. Returns the offset of the resulting table.
-flatbuffers::uoffset_t WriteTable(FlatbufferType type,
- const ::std::vector<FieldElement> &elements,
- flatbuffers::FlatBufferBuilder *fbb) {
- // End of a nested struct! Add it.
- const flatbuffers::uoffset_t start = fbb->StartTable();
+std::optional<Element> WriteObject(FlatbufferType type,
+ const ::std::vector<FieldElement> &elements,
+ flatbuffers::FlatBufferBuilder *fbb) {
+ // End of a nested object! Add it.
+ if (type.IsTable()) {
+ const flatbuffers::uoffset_t start = fbb->StartTable();
- ::std::vector<bool> fields_in_use(type.NumberFields(), false);
+ ::std::vector<bool> fields_in_use(type.NumberFields(), false);
- for (const FieldElement &field_element : elements) {
- AddSingleElement(type, field_element, &fields_in_use, fbb);
+ for (const FieldElement &field_element : elements) {
+ AddSingleElement(type, field_element, &fields_in_use, fbb);
+ }
+
+ return Element{
+ flatbuffers::Offset<flatbuffers::String>{fbb->EndTable(start)}};
+ } else if (type.IsStruct()) {
+ // In order to write an inline struct, we need to fill out each field at the
+ // correct position inline in memory. In order to do this, we retrieve the
+ // offset/size of each field, and directly populate that memory with the
+ // relevant value.
+ std::vector<uint8_t> buffer(type.InlineSize(), 0);
+ for (size_t field_index = 0;
+ field_index < static_cast<size_t>(type.NumberFields());
+ ++field_index) {
+ auto it = std::find_if(elements.begin(), elements.end(),
+ [field_index](const FieldElement &field) {
+ return field.field_index ==
+ static_cast<int>(field_index);
+ });
+ if (it == elements.end()) {
+ fprintf(stderr,
+ "All fields must be specified for struct types (field %s "
+ "missing).\n",
+ type.FieldName(field_index).data());
+ return std::nullopt;
+ }
+
+ uint8_t *field_data = buffer.data() + type.StructFieldOffset(field_index);
+ const size_t field_size = type.FieldInlineSize(field_index);
+ switch (it->element.type) {
+ case Element::ElementType::INT:
+ if (!SetStructElement(type, field_index, it->element.int_element,
+ field_data)) {
+ return std::nullopt;
+ }
+ break;
+ case Element::ElementType::DOUBLE:
+ if (!SetStructElement(type, field_index, it->element.double_element,
+ field_data)) {
+ return std::nullopt;
+ }
+ break;
+ case Element::ElementType::STRUCT:
+ CHECK_EQ(field_size, it->element.struct_data.size());
+ memcpy(field_data, it->element.struct_data.data(), field_size);
+ break;
+ case Element::ElementType::OFFSET:
+ LOG(FATAL)
+ << "This should be unreachable; structs cannot contain offsets.";
+ break;
+ }
+ }
+ return Element{buffer};
}
-
- return fbb->EndTable(start);
+ LOG(FATAL) << "Unimplemented.";
}
// Class to parse JSON into a flatbuffer.
@@ -135,7 +298,7 @@
// Adds *_value for the provided field. If we are in a vector, queues the
// data up in vector_elements. Returns true on success.
- bool AddElement(int field_index, int64_t int_value);
+ bool AddElement(int field_index, absl::int128 int_value);
bool AddElement(int field_index, double double_value);
bool AddElement(int field_index, const ::std::string &data);
@@ -144,12 +307,13 @@
// Pushes an element as part of a vector. Returns true on success.
bool PushElement(flatbuffers::ElementaryType elementary_type,
- int64_t int_value);
+ absl::int128 int_value);
bool PushElement(flatbuffers::ElementaryType elementary_type,
double double_value);
bool PushElement(flatbuffers::ElementaryType elementary_type,
flatbuffers::Offset<flatbuffers::String> offset_value);
-
+ bool PushElement(const FlatbufferType &type,
+ const std::vector<uint8_t> &struct_data);
flatbuffers::FlatBufferBuilder *fbb_;
// This holds the state information that is needed as you recurse into
@@ -170,8 +334,8 @@
// For scalar types (not strings, and not nested tables), the vector ends
// up being implemented as a start and end, and a block of data. So we
// can't just push offsets in as we go. We either need to reproduce the
- // logic inside flatbuffers, or build up vectors of the data. Vectors will
- // be a bit of extra stack space, but whatever.
+ // logic inside flatbuffers, or build up vectors of the data. Vectors
+ // will be a bit of extra stack space, but whatever.
//
// Strings and nested structures are vectors of offsets.
// into the vector. Once you get to the end, you build up a vector and
@@ -238,17 +402,23 @@
fprintf(stderr, "Empty stack\n");
return false;
} else {
- // End of a nested struct! Add it.
- const flatbuffers::uoffset_t end =
- WriteTable(stack_.back().type, stack_.back().elements, fbb_);
+ // End of a nested object! Add it.
+ std::optional<Element> object =
+ WriteObject(stack_.back().type, stack_.back().elements, fbb_);
+ if (!object.has_value()) {
+ return false;
+ }
// We now want to talk about the parent structure. Pop the child.
stack_.pop_back();
if (stack_.size() == 0) {
+ CHECK_EQ(static_cast<int>(object->type),
+ static_cast<int>(Element::ElementType::OFFSET))
+ << ": JSON parsing only supports parsing flatbuffer tables.";
// Instead of queueing it up in the stack, return it through the
// passed in variable.
- *table_end = end;
+ *table_end = object->offset_element.o;
} else {
// And now we can add it.
const int field_index = stack_.back().field_index;
@@ -256,10 +426,10 @@
// Do the right thing if we are in a vector.
if (in_vector()) {
stack_.back().vector_elements.emplace_back(
- flatbuffers::Offset<flatbuffers::String>(end));
+ std::move(object.value()));
} else {
- stack_.back().elements.emplace_back(
- field_index, flatbuffers::Offset<flatbuffers::String>(end));
+ stack_.back().elements.emplace_back(field_index,
+ std::move(object.value()));
}
}
}
@@ -294,7 +464,7 @@
case Tokenizer::TokenType::kNumberValue: {
bool is_int = true;
double double_value;
- long long int_value;
+ absl::int128 int_value;
if (token == Tokenizer::TokenType::kTrueValue) {
int_value = 1;
} else if (token == Tokenizer::TokenType::kFalseValue) {
@@ -313,7 +483,7 @@
if (is_int) {
// No need to get too stressed about bool vs int. Convert them all.
- int64_t val = int_value;
+ absl::int128 val = int_value;
if (!AddElement(field_index, val)) return false;
} else {
if (!AddElement(field_index, double_value)) return false;
@@ -342,7 +512,7 @@
return false;
}
-bool JsonParser::AddElement(int field_index, int64_t int_value) {
+bool JsonParser::AddElement(int field_index, absl::int128 int_value) {
if (stack_.back().type.FieldIsRepeating(field_index) != in_vector()) {
fprintf(stderr, "Type and json disagree on if we are in a vector or not\n");
return false;
@@ -393,7 +563,7 @@
const FlatbufferType enum_type = type.FieldType(field_index);
CHECK(enum_type.IsEnum());
- const std::optional<int64_t> int_value = enum_type.EnumValue(data);
+ const std::optional<absl::int128> int_value = enum_type.EnumValue(data);
if (!int_value) {
const std::string_view name = type.FieldName(field_index);
@@ -448,11 +618,15 @@
case Element::ElementType::OFFSET:
return AddSingleElement(type, field_element.field_index,
field_element.element.offset_element, fbb);
+ case Element::ElementType::STRUCT:
+ return AddSingleElement(type, field_element.field_index,
+ field_element.element.struct_data, fbb);
}
return false;
}
-bool AddSingleElement(FlatbufferType type, int field_index, int64_t int_value,
+bool AddSingleElement(FlatbufferType type, int field_index,
+ absl::int128 int_value,
flatbuffers::FlatBufferBuilder *fbb
) {
@@ -463,37 +637,39 @@
type.FieldElementaryType(field_index);
switch (elementary_type) {
case flatbuffers::ET_BOOL:
- fbb->AddElement<bool>(field_offset, int_value);
+ fbb->AddElement<bool>(field_offset, static_cast<bool>(int_value));
return true;
case flatbuffers::ET_CHAR:
- fbb->AddElement<int8_t>(field_offset, int_value);
+ fbb->AddElement<int8_t>(field_offset, static_cast<int8_t>(int_value));
return true;
case flatbuffers::ET_UCHAR:
- fbb->AddElement<uint8_t>(field_offset, int_value);
+ fbb->AddElement<uint8_t>(field_offset, static_cast<uint8_t>(int_value));
return true;
case flatbuffers::ET_SHORT:
- fbb->AddElement<int16_t>(field_offset, int_value);
+ fbb->AddElement<int16_t>(field_offset, static_cast<int16_t>(int_value));
return true;
case flatbuffers::ET_USHORT:
- fbb->AddElement<uint16_t>(field_offset, int_value);
+ fbb->AddElement<uint16_t>(field_offset, static_cast<uint16_t>(int_value));
return true;
case flatbuffers::ET_INT:
- fbb->AddElement<int32_t>(field_offset, int_value);
+ fbb->AddElement<int32_t>(field_offset, static_cast<int32_t>(int_value));
return true;
case flatbuffers::ET_UINT:
- fbb->AddElement<uint32_t>(field_offset, int_value);
+ fbb->AddElement<uint32_t>(field_offset, static_cast<uint32_t>(int_value));
return true;
case flatbuffers::ET_LONG:
- fbb->AddElement<int64_t>(field_offset, int_value);
+ fbb->AddElement<int64_t>(field_offset, static_cast<int64_t>(int_value));
return true;
case flatbuffers::ET_ULONG:
- fbb->AddElement<uint64_t>(field_offset, int_value);
+ fbb->AddElement<uint64_t>(field_offset, static_cast<uint64_t>(int_value));
return true;
+ // The floating point cases occur when someone specifies an integer in the
+ // JSON for a double field.
case flatbuffers::ET_FLOAT:
- fbb->AddElement<float>(field_offset, int_value);
+ fbb->AddElement<float>(field_offset, static_cast<float>(int_value));
return true;
case flatbuffers::ET_DOUBLE:
- fbb->AddElement<double>(field_offset, int_value);
+ fbb->AddElement<double>(field_offset, static_cast<double>(int_value));
return true;
case flatbuffers::ET_STRING:
case flatbuffers::ET_UTYPE:
@@ -545,6 +721,7 @@
}
return false;
}
+
bool AddSingleElement(FlatbufferType type, int field_index,
flatbuffers::Offset<flatbuffers::String> offset_element,
flatbuffers::FlatBufferBuilder *fbb) {
@@ -587,11 +764,27 @@
return false;
}
+bool AddSingleElement(FlatbufferType type, int field_index,
+ const std::vector<uint8_t> &data,
+ flatbuffers::FlatBufferBuilder *fbb) {
+ // Structs are always inline.
+ // We have to do somewhat manual serialization to get the struct into place,
+ // since the regular FlatBufferBuilder assumes that you will know the type of
+ // the struct that you are constructing at compile time.
+ fbb->Align(type.FieldType(field_index).Alignment());
+ fbb->PushBytes(data.data(), data.size());
+ fbb->AddStructOffset(flatbuffers::FieldIndexToOffset(
+ static_cast<flatbuffers::voffset_t>(field_index)),
+ fbb->GetSize());
+ return true;
+}
+
bool JsonParser::FinishVector(int field_index) {
// Vectors have a start (unfortunately which needs to know the size)
const size_t inline_size = stack_.back().type.FieldInlineSize(field_index);
+ const size_t alignment = stack_.back().type.FieldInlineAlignment(field_index);
fbb_->StartVector(stack_.back().vector_elements.size(), inline_size,
- /*align=*/inline_size);
+ /*align=*/alignment);
const flatbuffers::ElementaryType elementary_type =
stack_.back().type.FieldElementaryType(field_index);
@@ -609,6 +802,11 @@
case Element::ElementType::OFFSET:
if (!PushElement(elementary_type, element.offset_element)) return false;
break;
+ case Element::ElementType::STRUCT:
+ if (!PushElement(stack_.back().type.FieldType(field_index),
+ element.struct_data))
+ return false;
+ break;
}
}
@@ -621,40 +819,40 @@
}
bool JsonParser::PushElement(flatbuffers::ElementaryType elementary_type,
- int64_t int_value) {
+ absl::int128 int_value) {
switch (elementary_type) {
case flatbuffers::ET_BOOL:
- fbb_->PushElement<bool>(int_value);
+ fbb_->PushElement<bool>(static_cast<bool>(int_value));
return true;
case flatbuffers::ET_CHAR:
- fbb_->PushElement<int8_t>(int_value);
+ fbb_->PushElement<int8_t>(static_cast<int8_t>(int_value));
return true;
case flatbuffers::ET_UCHAR:
- fbb_->PushElement<uint8_t>(int_value);
+ fbb_->PushElement<uint8_t>(static_cast<uint8_t>(int_value));
return true;
case flatbuffers::ET_SHORT:
- fbb_->PushElement<int16_t>(int_value);
+ fbb_->PushElement<int16_t>(static_cast<int16_t>(int_value));
return true;
case flatbuffers::ET_USHORT:
- fbb_->PushElement<uint16_t>(int_value);
+ fbb_->PushElement<uint16_t>(static_cast<uint16_t>(int_value));
return true;
case flatbuffers::ET_INT:
- fbb_->PushElement<int32_t>(int_value);
+ fbb_->PushElement<int32_t>(static_cast<int32_t>(int_value));
return true;
case flatbuffers::ET_UINT:
- fbb_->PushElement<uint32_t>(int_value);
+ fbb_->PushElement<uint32_t>(static_cast<uint32_t>(int_value));
return true;
case flatbuffers::ET_LONG:
- fbb_->PushElement<int64_t>(int_value);
+ fbb_->PushElement<int64_t>(static_cast<int64_t>(int_value));
return true;
case flatbuffers::ET_ULONG:
- fbb_->PushElement<uint64_t>(int_value);
+ fbb_->PushElement<uint64_t>(static_cast<uint64_t>(int_value));
return true;
case flatbuffers::ET_FLOAT:
- fbb_->PushElement<float>(int_value);
+ fbb_->PushElement<float>(static_cast<float>(int_value));
return true;
case flatbuffers::ET_DOUBLE:
- fbb_->PushElement<double>(int_value);
+ fbb_->PushElement<double>(static_cast<double>(int_value));
return true;
case flatbuffers::ET_STRING:
case flatbuffers::ET_UTYPE:
@@ -698,6 +896,17 @@
return false;
}
+bool JsonParser::PushElement(const FlatbufferType &type,
+ const std::vector<uint8_t> &struct_data) {
+ // To add a struct to a vector, we just need to get the relevant bytes pushed
+ // straight into the builder. The FlatBufferBuilder normally expects that you
+ // will know the type of your struct at compile-time, so doesn't have a
+ // first-class way to do this.
+ fbb_->Align(type.Alignment());
+ fbb_->PushBytes(struct_data.data(), struct_data.size());
+ return true;
+}
+
bool JsonParser::PushElement(
flatbuffers::ElementaryType elementary_type,
flatbuffers::Offset<flatbuffers::String> offset_value) {
diff --git a/aos/json_to_flatbuffer.fbs b/aos/json_to_flatbuffer.fbs
index 7180322..e66ec4a 100644
--- a/aos/json_to_flatbuffer.fbs
+++ b/aos/json_to_flatbuffer.fbs
@@ -62,6 +62,15 @@
nested_struct:FooStructNested;
}
+struct ScalarSweepStruct {
+ foo_float:float;
+ foo_double:double;
+ foo_int32:int32;
+ foo_uint32:uint32;
+ foo_int64:int64;
+ foo_uint64:uint64;
+}
+
struct StructEnum {
foo_enum:BaseType;
}
@@ -131,9 +140,11 @@
foo_struct:FooStruct (id: 34);
vector_foo_struct:[FooStruct] (id: 35);
foo_struct_enum:StructEnum (id: 36);
+ foo_struct_scalars:ScalarSweepStruct (id: 37);
+ vector_foo_struct_scalars:[ScalarSweepStruct] (id: 38);
- foo_enum_nonconsecutive:NonConsecutive (id: 37);
- foo_enum_nonconsecutive_default:NonConsecutive = Big (id: 38);
+ foo_enum_nonconsecutive:NonConsecutive (id: 39);
+ foo_enum_nonconsecutive_default:NonConsecutive = Big (id: 40);
}
root_type Configuration;
diff --git a/aos/json_to_flatbuffer_test.cc b/aos/json_to_flatbuffer_test.cc
index a145364..4900711 100644
--- a/aos/json_to_flatbuffer_test.cc
+++ b/aos/json_to_flatbuffer_test.cc
@@ -87,6 +87,62 @@
EXPECT_TRUE(JsonAndBack("{ \"foo_enum_nonconsecutive\": \"Big\" }"));
}
+TEST_F(JsonToFlatbufferTest, Structs) {
+ EXPECT_TRUE(
+ JsonAndBack("{ \"foo_struct\": { \"foo_byte\": 1, \"nested_struct\": { "
+ "\"foo_byte\": 2 } } }"));
+ EXPECT_TRUE(JsonAndBack(
+ "{ \"foo_struct_scalars\": { \"foo_float\": 1.234, \"foo_double\": "
+ "4.567, \"foo_int32\": -971, \"foo_uint32\": 4294967294, \"foo_int64\": "
+ "-1030, \"foo_uint64\": 18446744073709551614 } }"));
+ // Confirm that we parse integers into floating point fields correctly.
+ EXPECT_TRUE(JsonAndBack(
+ "{ \"foo_struct_scalars\": { \"foo_float\": 1, \"foo_double\": "
+ "2, \"foo_int32\": 3, \"foo_uint32\": 4, \"foo_int64\": "
+ "5, \"foo_uint64\": 6 } }",
+ "{ \"foo_struct_scalars\": { \"foo_float\": 1.0, \"foo_double\": "
+ "2.0, \"foo_int32\": 3, \"foo_uint32\": 4, \"foo_int64\": "
+ "5, \"foo_uint64\": 6 } }"));
+ EXPECT_TRUE(JsonAndBack(
+ "{ \"vector_foo_struct_scalars\": [ { \"foo_float\": 1.234, "
+ "\"foo_double\": 4.567, \"foo_int32\": -971, \"foo_uint32\": 4294967294, "
+ "\"foo_int64\": -1030, \"foo_uint64\": 18446744073709551614 }, { "
+ "\"foo_float\": 2.0, \"foo_double\": 4.1, \"foo_int32\": 10, "
+ "\"foo_uint32\": 13, \"foo_int64\": 15, \"foo_uint64\": 18 } ] }"));
+ EXPECT_TRUE(
+ JsonAndBack("{ \"foo_struct_enum\": { \"foo_enum\": \"UByte\" } }"));
+ EXPECT_TRUE(
+ JsonAndBack("{ \"vector_foo_struct\": [ { \"foo_byte\": 1, "
+ "\"nested_struct\": { \"foo_byte\": 2 } } ] }"));
+ EXPECT_TRUE(JsonAndBack(
+ "{ \"vector_foo_struct\": [ { \"foo_byte\": 1, \"nested_struct\": { "
+ "\"foo_byte\": 2 } }, { \"foo_byte\": 3, \"nested_struct\": { "
+ "\"foo_byte\": 4 } }, { \"foo_byte\": 5, \"nested_struct\": { "
+ "\"foo_byte\": 6 } } ] }"));
+}
+
+// Confirm that we correctly die when input JSON is missing fields inside of a
+// struct.
+TEST_F(JsonToFlatbufferTest, StructMissingField) {
+ ::testing::internal::CaptureStderr();
+ EXPECT_FALSE(
+ JsonAndBack("{ \"foo_struct\": { \"nested_struct\": { "
+ "\"foo_byte\": 2 } } }"));
+ EXPECT_FALSE(JsonAndBack(
+ "{ \"foo_struct\": { \"foo_byte\": 1, \"nested_struct\": { } } }"));
+ EXPECT_FALSE(JsonAndBack("{ \"foo_struct\": { \"foo_byte\": 1 } }"));
+ std::string output = ::testing::internal::GetCapturedStderr();
+ EXPECT_EQ(
+ R"output(All fields must be specified for struct types (field foo_byte missing).
+All fields must be specified for struct types (field foo_byte missing).
+All fields must be specified for struct types (field foo_byte missing).
+All fields must be specified for struct types (field foo_byte missing).
+All fields must be specified for struct types (field nested_struct missing).
+All fields must be specified for struct types (field nested_struct missing).
+)output",
+ output);
+}
+
// Tests that Inf is handled correctly
TEST_F(JsonToFlatbufferTest, Inf) {
EXPECT_TRUE(JsonAndBack("{ \"foo_float\": inf }"));
@@ -209,7 +265,7 @@
}
// Test nested messages, and arrays of nested messages.
-TEST_F(JsonToFlatbufferTest, NestedStruct) {
+TEST_F(JsonToFlatbufferTest, NestedTable) {
EXPECT_TRUE(
JsonAndBack("{ \"single_application\": { \"name\": \"woot\" } }"));
diff --git a/aos/json_tokenizer.cc b/aos/json_tokenizer.cc
index eab7fcc..d277c1e 100644
--- a/aos/json_tokenizer.cc
+++ b/aos/json_tokenizer.cc
@@ -531,14 +531,9 @@
return TokenType::kError;
}
-bool Tokenizer::FieldAsInt(long long *value) {
+bool Tokenizer::FieldAsInt(absl::int128 *value) {
const char *pos = field_value().c_str();
- errno = 0;
- *value = strtoll(field_value().c_str(), const_cast<char **>(&pos), 10);
- if (pos != field_value().c_str() + field_value().size() || errno != 0) {
- return false;
- }
- return true;
+ return absl::SimpleAtoi(pos, value);
}
bool Tokenizer::FieldAsDouble(double *value) {
diff --git a/aos/json_tokenizer.h b/aos/json_tokenizer.h
index 892c575..ab928f4 100644
--- a/aos/json_tokenizer.h
+++ b/aos/json_tokenizer.h
@@ -5,6 +5,7 @@
#include <string_view>
#include <vector>
+#include "absl/strings/numbers.h"
#include "flatbuffers/util.h"
namespace aos {
@@ -41,7 +42,7 @@
// Parses the current field value as a long long. Returns false if it failed
// to parse.
- bool FieldAsInt(long long *value);
+ bool FieldAsInt(absl::int128 *value);
// Parses the current field value as a double. Returns false if it failed
// to parse.
bool FieldAsDouble(double *value);