Merge commit '6dccd64de51ea960cb6f0d975768c874814b4c75' into HEAD
Update flatbuffers. Relevant merge tasks:
* A field got added to reflection.fbs; this had fallout in
reflection_generated.h and the logger_test shas.
* Resolved merge conflict in rust/flatbuffers/src/lib.rs
* Reverted upstream change that made Table rust struct members private.
* FlatBufferBuilder Create*Vector calls now include alignment.
* nim codegen got added; needed to update to use scoped enums.
Main fix that motivated this update is
https://github.com/google/flatbuffers/pull/7588
Change-Id: I6bbe5d56846f426fa5f2a82c4f2bc77be2b93bb0
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/third_party/flatbuffers/src/idl_parser.cpp b/third_party/flatbuffers/src/idl_parser.cpp
index 524c996..48d0f07 100644
--- a/third_party/flatbuffers/src/idl_parser.cpp
+++ b/third_party/flatbuffers/src/idl_parser.cpp
@@ -507,6 +507,8 @@
case ')':
case '[':
case ']':
+ case '<':
+ case '>':
case ',':
case ':':
case ';':
@@ -1584,7 +1586,6 @@
return NoError();
}
-
CheckedError Parser::ParseAlignAttribute(const std::string &align_constant,
size_t min_align, size_t *align) {
// Use uint8_t to avoid problems with size_t==`unsigned long` on LP64.
@@ -1614,6 +1615,7 @@
});
ECHECK(err);
+ const size_t alignment = InlineAlignment(type);
const size_t len = count * InlineSize(type) / InlineAlignment(type);
const size_t elemsize = InlineAlignment(type);
const auto force_align = field->attributes.Lookup("force_align");
@@ -1623,7 +1625,8 @@
if (align > 1) { builder_.ForceVectorAlignment(len, elemsize, align); }
}
- builder_.StartVector(len, elemsize);
+ // TODO Fix using element alignment as size (`elemsize`)!
+ builder_.StartVector(len, elemsize, alignment);
for (uoffset_t i = 0; i < count; i++) {
// start at the back, since we're building the data backwards.
auto &val = field_stack_.back().first;
@@ -2448,14 +2451,17 @@
EXPECT(kTokenIntegerConstant);
}
- ECHECK(evb.AcceptEnumerator());
-
if (opts.proto_mode && Is('[')) {
NEXT();
// ignore attributes on enums.
while (token_ != ']') NEXT();
NEXT();
+ } else {
+ // parse attributes in fbs schema
+ ECHECK(ParseMetaData(&ev.attributes));
}
+
+ ECHECK(evb.AcceptEnumerator());
}
if (!Is(opts.proto_mode ? ';' : ',')) break;
NEXT();
@@ -2550,7 +2556,8 @@
IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster |
IDLOptions::kKotlin | IDLOptions::kCpp | IDLOptions::kJava |
IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary |
- IDLOptions::kGo | IDLOptions::kPython | IDLOptions::kJson;
+ IDLOptions::kGo | IDLOptions::kPython | IDLOptions::kJson |
+ IDLOptions::kNim;
unsigned long langs = opts.lang_to_generate;
return (langs > 0 && langs < IDLOptions::kMAX) && !(langs & ~supported_langs);
}
@@ -2561,7 +2568,7 @@
bool Parser::SupportsDefaultVectorsAndStrings() const {
static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
- IDLOptions::kRust | IDLOptions::kSwift;
+ IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kNim;
return !(opts.lang_to_generate & ~supported_langs);
}
@@ -2569,14 +2576,14 @@
return (opts.lang_to_generate &
~(IDLOptions::kCpp | IDLOptions::kTs | IDLOptions::kPhp |
IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kKotlin |
- IDLOptions::kBinary | IDLOptions::kSwift)) == 0;
+ IDLOptions::kBinary | IDLOptions::kSwift | IDLOptions::kNim)) == 0;
}
bool Parser::SupportsAdvancedArrayFeatures() const {
return (opts.lang_to_generate &
~(IDLOptions::kCpp | IDLOptions::kPython | IDLOptions::kJava |
IDLOptions::kCSharp | IDLOptions::kJsonSchema | IDLOptions::kJson |
- IDLOptions::kBinary | IDLOptions::kRust)) == 0;
+ IDLOptions::kBinary | IDLOptions::kRust | IDLOptions::kTs)) == 0;
}
Namespace *Parser::UniqueNamespace(Namespace *ns) {
@@ -2891,6 +2898,8 @@
NEXT();
while (!Is(';')) { NEXT(); } // A variety of formats, just skip.
NEXT();
+ } else if (IsIdent("map")) {
+ ECHECK(ParseProtoMapField(struct_def));
} else {
std::vector<std::string> field_comment = doc_comment_;
// Parse the qualifier.
@@ -3025,6 +3034,41 @@
return NoError();
}
+CheckedError Parser::ParseProtoMapField(StructDef *struct_def) {
+ NEXT();
+ EXPECT('<');
+ Type key_type;
+ ECHECK(ParseType(key_type));
+ EXPECT(',');
+ Type value_type;
+ ECHECK(ParseType(value_type));
+ EXPECT('>');
+ auto field_name = attribute_;
+ NEXT();
+ EXPECT('=');
+ EXPECT(kTokenIntegerConstant);
+ EXPECT(';');
+
+ auto entry_table_name = ConvertCase(field_name, Case::kUpperCamel) + "Entry";
+ StructDef *entry_table;
+ ECHECK(StartStruct(entry_table_name, &entry_table));
+ entry_table->has_key = true;
+ FieldDef *key_field;
+ ECHECK(AddField(*entry_table, "key", key_type, &key_field));
+ key_field->key = true;
+ FieldDef *value_field;
+ ECHECK(AddField(*entry_table, "value", value_type, &value_field));
+
+ Type field_type;
+ field_type.base_type = BASE_TYPE_VECTOR;
+ field_type.element = BASE_TYPE_STRUCT;
+ field_type.struct_def = entry_table;
+ FieldDef *field;
+ ECHECK(AddField(*struct_def, field_name, field_type, &field));
+
+ return NoError();
+}
+
CheckedError Parser::ParseProtoKey() {
if (token_ == '(') {
NEXT();
@@ -3259,6 +3303,10 @@
return done;
}
+std::ptrdiff_t Parser::BytesConsumed() const {
+ return std::distance(source_, cursor_);
+}
+
CheckedError Parser::StartParseFile(const char *source,
const char *source_filename) {
file_being_parsed_ = source_filename ? source_filename : "";
@@ -3413,7 +3461,6 @@
return NoError();
}
-
CheckedError Parser::DoParse(const char *source, const char **include_paths,
const char *source_filename,
const char *include_filename) {
@@ -3601,9 +3648,11 @@
: nullptr);
}
}
- // Check that JSON file doesn't contain more objects or IDL directives.
- // Comments after JSON are allowed.
- EXPECT(kTokenEof);
+ if (opts.require_json_eof) {
+ // Check that JSON file doesn't contain more objects or IDL directives.
+ // Comments after JSON are allowed.
+ EXPECT(kTokenEof);
+ }
return NoError();
}
@@ -3635,6 +3684,44 @@
// Schema serialization functionality:
+static flatbuffers::Offset<
+ flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
+SerializeAttributesCommon(const SymbolTable<Value> &attributes,
+ FlatBufferBuilder *builder, const Parser &parser) {
+ std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
+ for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
+ auto it = parser.known_attributes_.find(kv->first);
+ FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
+ if (parser.opts.binary_schema_builtins || !it->second) {
+ auto key = builder->CreateString(kv->first);
+ auto val = builder->CreateString(kv->second->constant);
+ attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
+ }
+ }
+ if (attrs.size()) {
+ return builder->CreateVectorOfSortedTables(&attrs);
+ } else {
+ return 0;
+ }
+}
+
+static bool DeserializeAttributesCommon(
+ SymbolTable<Value> &attributes, Parser &parser,
+ const Vector<Offset<reflection::KeyValue>> *attrs) {
+ if (attrs == nullptr) return true;
+ for (uoffset_t i = 0; i < attrs->size(); ++i) {
+ auto kv = attrs->Get(i);
+ auto value = new Value();
+ if (kv->value()) { value->constant = kv->value()->str(); }
+ if (attributes.Add(kv->key()->str(), value)) {
+ delete value;
+ return false;
+ }
+ parser.known_attributes_[kv->key()->str()];
+ }
+ return true;
+}
+
void Parser::Serialize() {
builder_.Clear();
AssignIndices(structs_.vec);
@@ -3927,32 +4014,52 @@
return true;
}
-Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
- const Parser &parser) const {
- auto name__ = builder->CreateString(name);
- auto type__ = union_type.Serialize(builder);
- auto docs__ = parser.opts.binary_schema_comments
- ? builder->CreateVectorOfStrings(doc_comment)
- : 0;
- return reflection::CreateEnumVal(*builder, name__, value, type__, docs__);
+flatbuffers::Offset<
+ flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
+EnumVal::SerializeAttributes(FlatBufferBuilder *builder,
+ const Parser &parser) const {
+ return SerializeAttributesCommon(attributes, builder, parser);
}
-bool EnumVal::Deserialize(const Parser &parser,
- const reflection::EnumVal *val) {
+bool EnumVal::DeserializeAttributes(
+ Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
+ return DeserializeAttributesCommon(attributes, parser, attrs);
+}
+
+Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
+ const Parser &parser) const {
+ const auto name__ = builder->CreateString(name);
+ const auto type__ = union_type.Serialize(builder);
+ const auto attr__ = SerializeAttributes(builder, parser);
+ const auto docs__ = parser.opts.binary_schema_comments
+ ? builder->CreateVectorOfStrings(doc_comment)
+ : 0;
+ return reflection::CreateEnumVal(*builder, name__, value, type__, docs__,
+ attr__);
+}
+
+bool EnumVal::Deserialize(Parser &parser, const reflection::EnumVal *val) {
name = val->name()->str();
value = val->value();
if (!union_type.Deserialize(parser, val->union_type())) return false;
+ if (!DeserializeAttributes(parser, val->attributes())) return false;
DeserializeDoc(doc_comment, val->documentation());
return true;
}
Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
+ size_t element_size = SizeOf(element);
+ if (base_type == BASE_TYPE_VECTOR && element == BASE_TYPE_STRUCT &&
+ struct_def->bytesize != 0) {
+ // struct_def->bytesize==0 means struct is table
+ element_size = struct_def->bytesize;
+ }
return reflection::CreateType(
*builder, static_cast<reflection::BaseType>(base_type),
static_cast<reflection::BaseType>(element),
struct_def ? struct_def->index : (enum_def ? enum_def->index : -1),
fixed_length, static_cast<uint32_t>(SizeOf(base_type)),
- static_cast<uint32_t>(SizeOf(element)));
+ static_cast<uint32_t>(element_size));
}
bool Type::Deserialize(const Parser &parser, const reflection::Type *type) {
@@ -3986,37 +4093,12 @@
flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
Definition::SerializeAttributes(FlatBufferBuilder *builder,
const Parser &parser) const {
- std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
- for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
- auto it = parser.known_attributes_.find(kv->first);
- FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
- if (parser.opts.binary_schema_builtins || !it->second) {
- auto key = builder->CreateString(kv->first);
- auto val = builder->CreateString(kv->second->constant);
- attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
- }
- }
- if (attrs.size()) {
- return builder->CreateVectorOfSortedTables(&attrs);
- } else {
- return 0;
- }
+ return SerializeAttributesCommon(attributes, builder, parser);
}
bool Definition::DeserializeAttributes(
Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
- if (attrs == nullptr) return true;
- for (uoffset_t i = 0; i < attrs->size(); ++i) {
- auto kv = attrs->Get(i);
- auto value = new Value();
- if (kv->value()) { value->constant = kv->value()->str(); }
- if (attributes.Add(kv->key()->str(), value)) {
- delete value;
- return false;
- }
- parser.known_attributes_[kv->key()->str()];
- }
- return true;
+ return DeserializeAttributesCommon(attributes, parser, attrs);
}
/************************************************************************/
@@ -4131,17 +4213,19 @@
struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name);
auto struct_def_base = base.LookupStruct(qualified_name);
if (!struct_def_base) continue;
+ std::set<FieldDef *> renamed_fields;
for (auto fit = struct_def.fields.vec.begin();
fit != struct_def.fields.vec.end(); ++fit) {
auto &field = **fit;
auto field_base = struct_def_base->fields.Lookup(field.name);
+ const auto qualified_field_name = qualified_name + "." + field.name;
if (field_base) {
if (field.value.offset != field_base->value.offset)
- return "offsets differ for field: " + field.name;
+ return "offsets differ for field: " + qualified_field_name;
if (field.value.constant != field_base->value.constant)
- return "defaults differ for field: " + field.name;
+ return "defaults differ for field: " + qualified_field_name;
if (!EqualByName(field.value.type, field_base->value.type))
- return "types differ for field: " + field.name;
+ return "types differ for field: " + qualified_field_name;
} else {
// Doesn't have to exist, deleting fields is fine.
// But we should check if there is a field that has the same offset
@@ -4150,14 +4234,28 @@
fbit != struct_def_base->fields.vec.end(); ++fbit) {
field_base = *fbit;
if (field.value.offset == field_base->value.offset) {
+ renamed_fields.insert(field_base);
if (!EqualByName(field.value.type, field_base->value.type))
- return "field renamed to different type: " + field.name;
+ return "field renamed to different type: " + qualified_field_name;
break;
}
}
}
}
+ // deletion of trailing fields are not allowed
+ for (auto fit = struct_def_base->fields.vec.begin();
+ fit != struct_def_base->fields.vec.end(); ++fit) {
+ auto &field_base = **fit;
+ // not a renamed field
+ if (renamed_fields.find(&field_base) == renamed_fields.end()) {
+ auto field = struct_def.fields.Lookup(field_base.name);
+ if (!field) {
+ return "field deleted: " + qualified_name + "." + field_base.name;
+ }
+ }
+ }
}
+
for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) {
auto &enum_def = **eit;
auto qualified_name =