Squashed 'third_party/flatbuffers/' changes from e5f331db9..bc44fad35
bc44fad35 UnPackTo disable merge by default (#7527)
4fca4dc60 [TS/JS] Move TS tests to dedicated folder and deps upgrade (#7508)
036032373 Bump junit from 4.13 to 4.13.1 in /java (#7526)
89dfb43f3 Replace `bash JavaTest.sh` with `mvn test` (#7500)
c49aff4b6 enabled cpp17 tests in CI (#7524)
56e60223c prevent force_align attribute on enums (#7523)
89b1f5aa1 remove travis config (#7522)
b90159823 [Java][Flexbuffers] Add API to add nullables into the buffer. (#7521)
8cdc6a288 Install BuildFlatBuffers.cmake (#7519)
a67e35aff Moves all of the swift test code into tests/swift (#7509)
f124e41ae Updated Readme
4c954181c [Java][FlexBuffers] throwing exception for untyped fixed vectors (#7507)
7f7547737 [Android] Remove maven dependency of flatbuffers and use source folder (#7503)
a79d61ea8 Fixes issue with cocoapods failing to be published because of docc (#7505)
d465b39c3 [CMake]: fix breaking find_package change (#7499) (#7502)
c5a609dc2 [C#] Prepares for official Nuget release (#7496)
5634dc3d0 [ISSUE-6268] returns NaN insteadof nan (#7498)
37e37b8ca Updates cocoapods version (#7497)
8fd4534fb update android multidex setting (#7495)
d5427da52 Disable Android Build (#7494)
06c5c7ed0 FlatBuffers Version 2.0.8 (#7492)
b190ce11b Verifier Refinements (#7490)
bf5d23230 Namer applied to Typescript generator (#7488)
ce382d6dd [TS/JS] Add rollup and config to generate iife flatbuffers bundle (#7449)
41d9add7e C++: Add option to skip verifying nested flatbuffers (#7489)
6a8742754 [C++] support native_inline attribute for vector of tables (#7479)
694add668 Refactor test.cpp (#7487)
7edf8c908 Update scorecard to 1.1.2
b86387442 Fix typos (#7483)
e2eb5ee67 Include <array> head in stl_emulation.h (#7480)
994502b6d Version number in file package.json updated to 2.0.7 (#7476)
fa41e8367 [C++] Fixed crash when copying table with empty shared strings (#7477)
799cc8f7b Use type traits for specialization (#7475)
b7eb44147 Disable RTTI and rework use in idl_gen_ts.cpp (#7474)
8d01c5859 CMake project version detection made more robust (#7473)
237e8b71f Moved compiler warnings around (#7471)
eeb8fd60d Include builder.addOffset for vector of structs (#7470)
fef2ffc4d Use schema include name for keep-prefix (#7469)
8367664f1 Flatbuffers Version 2.0.7 (#7462)
d6f06c33f Reworked keep prefix (#7456)
627e8bf36 update grpc version (#7457)
883c42b7d disabling unpackto optimization (#7459)
7aae0af30 Remove old GRPC bash script and convert to python3 (#7454)
b057aa917 Grouped anonymous namespaces together, (#7455)
f1b26ff7f Change to GetTypeName (#7453)
9610a666b Generate SLSA signatures for Released zip files (#7450)
1e0f75a64 [WIP] speedup (#7452)
82b75407a Wrap types in namespace for --ts-flat-files and --gen-all (#7451)
f7c511957 Audit and fixups for GCC and Clang (#7212)
a66de58af Partial support for --ts-flat-files and --gen-all (#7446)
a3508f36d [Kotlin] Make sure namespace path exist for code generation (#7357)
137fec716 Stop using __has_trivial_copy on recent clang versions. (#7443)
214125e41 [C#] Rework how sorted vectors are looked up (#7441)
44a7dc999 Define minimum buffer size (#7440)
3cc2daa78 make_span overloads for pointer to vector (#7374) (#7435)
fa1174aa7 [TypeScript] Fix namespaceless schema generation (#7432)
83d4e2a10 Add checks to verifier (#7438)
8a09f3fb0 Fix FlexBuffers JS/TS bug https://github.com/google/flatbuffers/issues/6934 (#7434)
9dbe819ef Add flatc python tests to CI (#7437)
67c414958 Update TypeScriptTest.py to work better cross platform (#7436)
8b8c7dbdf Update gitingore to reflect name change (#7431)
2ee20a5f3 Remove auto including locale functions (#7430)
4be605604 [C++] Set StructDef::has_key property when deserializing from binary schema (#7386) (#7428)
fc5d86f1e [C++] Make template parameter in stl_emulation.h more explicit to avoid conflicts with cpprestsdk U macro (#7424)
9dce287ad Issue#6959 :Updated Automatically generated rust files. (#7425)
7798be3bb avoid zero-as-null-pointer warning (#7423)
966362e07 [C++] Vector of Tables equality (#7415)
a89c279ed [golang] Perform keyword escaping after case conversion (#7421)
a212b3c03 Turn of fail fast for C++ CI
9230f600d Remove stringop-overflow from error (#7422)
c79362156 [golang] Add support for text parsing with json struct tags (#7353)
ee2ced236 Moved TypeScriptTests to python script (#7411)
468c00a3f Rebased: grpc/compiler: Respect filename suffix and extension during code generation (#7414)
47c757f71 Add tests for flatc (#7405)
9a5ff8900 Add FLATBUFFERS_STRICT_MODE (#7408)
950444a34 [TS] Use TextEncoder and TextDecoder (#7400)
30d76198c Compilation issue msys2 #7399 (#7409)
cce3a66f0 Delete .travis directory
8d1cc6ac7 Revert "Compilation issue msys2 (#7403)" (#7407)
5b207639a Update readme.md
359e0f9d6 Revert "grpc/compiler: Respect filename suffix and extension during code generation (#7343)" (#7406)
ebbed0513 Delete cpp-linter.yml
aa395e5a5 (#7323) Rename CMake files according to project name (#7378)
32328075d Fix error msg format when generate GRPC failed (#7350)
97e89c5ac grpc/compiler: Respect filename suffix and extension during code generation (#7343)
5f6672be4 Fix Clang-Cl compile on Windows (#7308)
28e858c85 [TS/Bazel] Minor improvements to typescript.bzl (#7300)
987bebe67 [TS] fix incorrect reverse when writting array of structs (#7271)
ec0129369 Fix FlexBuffers Verifier tracking vectors reuse at wrong offset
50dd385b3 Add missing const (#7401)
da702cfd8 Compilation issue msys2 (#7403)
6e2791640 keep-prefix keeps relative pathing (#7394)
52fce5e53 fix(#7360): grpc used deprecated functions (#7361)
b7f13cd8e cpp_generator: comment out unused parameter to avoid warnings (#7381)
e42985e5a Updated Newtonsoft.Json to 13.0.1 (#7393)
0a8064637 Fix references to LICENSE file (#7377)
b9eea76a8 [Dart] Implement putBool to fix errors when serializing structs with bools (#7359)
1b9030015 Bump Newtonsoft.Json from 12.0.3 to 13.0.1 in /tests/FlatBuffers.Test (#7363)
83a43fc79 Reenable optional json (#7352)
5f0137602 Only include direct included filed (#7348)
9a1913a87 Revert "Implement optional scalars for JSON (#7322)" (#7351)
b4647beb8 Revert "Move reflection_ts_fbs into a separate directory (#7342)" (#7349)
d6060977a Remove asserting in verifier for flattests
987aa5b5e move -Wextra-semi to GCC 8.0+
42acdb63c [TS] Don't generate self-imports with --ts-flat-file (#7340)
0cc1aeb8c [golang] Create missing namespace directory structure (#7324) (#7325)
ba6c67170 [Kotlin] Remove download benchmark files dependency (#7314)
d2f33fc45 Disable Android on Linux CI build
0d1b72cbc [TS] fix ts import path issue (#7298)
9fce2fbf2 replace io/ioutil to os (#7281)
a18ea40d6 Implement optional scalars for JSON (#7322)
090caa280 Move reflection_ts_fbs into a separate directory (#7342)
49e1ea333 Implement optional scalars for Python (#7318)
11a198870 Started implementation for private flags in rust (#7269)
967df08b1 Adds full supposed for Wasm in the swift lib (#7328)
9aa08a429 Use keep case for Rust union discriminant type. (#7321)
9e8c758f5 Add explicit return types to lobster generated code (#7312)
74a25536b Add size check to fix out of bounds read risk (#7304)
12917af8a Update Rust docs page (#7296)
1ea2472f7 [swift] add had<ArrayName> property for arrays to check presence in a message (#7280)
0fe13cb28 Remove span ConstIterator/cbegin()/cend(). (#7295)
385dddc66 Namerkot (#7245)
750dde766 Make `flatc` generate Rust files not requiring `std` (#7273)
9917a168c [swift] Make swift module public (#7274)
76d3cca19 Rust: fix a name conflict when building with "no_std" feature (#7268)
c86e6d0e3 json inf parsing
d34dc32c2 fix include order
234d86c92 fixed off-by-one in parser
746c73b91 Add Annotations for Monster schema and example buffer
0bbfd4b2e fixes for annotator
716521953 Update readme.md (#7257)
a45f564cf [performance] Add aggressive systematic inlining in ByteBuffer and FlatBufferBuilder (#7253)
9d45a6403 more google merge fixes
ccfb4c20b Handle +/-inf in protos (#7256)
7bcd857b8 Specialize CreateVector with std::initializer_list (#7254)
23c8ab34c Swift update performance benchmark infrastructure (#7255)
70002dc5c various fixes for google merge
6e0e79f24 Add test for nested buffer verifier (#7252)
b856368d7 Turn off go modules temporary until we get a proper fix (#7251)
e37156a30 Keep the underlying storage capacity when clearing the FlatBufferBuilder. Gives a significant performance boost for serialisation of many small messages. (#7250)
a10b0e546 Java namer variable keep case (#7249)
275b73994 allow overriding FLATBUFFERS_MAX_ALIGNMENT
9d1ce9a10 Add parameter back to EndVector (#7246)
79afe6c3d Make Java namespaces keep case by default (#7243)
c6dbb2230 Add write permissions for labeller
18bacd3ea Expand test to make sure {}-initializers are properly understood by template. (#7242)
a2c913aec Add -Wnon-virtual-dtor
67b33b294 set workflows permissions to read-only (#7239)
7b5fd2bd0 [Kotlin] Fix key lookup returning null clashing with default value (#7237)
7181d7770 [Java] Fix key lookup returning null clashing with default value (#7236)
7f663b120 Allow CreateVectorOfStrings() to work with any string-type. (#7238)
173ebb694 Fixes a bug where the create function doesnt optional + required items (#7228)
d65823948 [Kotlin] Update gradle to 7.4.1 and simplify config files. (#7231)
ab4bf59e8 remove toascii (#7234)
eee44bbb2 disable cpp-linter (#7229)
a63fa51a1 Create cpp-linter.yml (#7208)
2049e5210 Adds a way to verify/exposes Entities ids (#7221)
832c618f5 Adds implementation flag for swift (#7202)
14615699f Started to migrate to target_compile_options (#7222)
20aad0c41 [C++] stl_emulation span::count_ is not const anymore (#7226) (#7227)
f083b33f2 code gen flexbuffer verifier (#7207)
bf17df346 [C++] generate sorted #include directives (#7213)
35281dedb Fix for [C++] flatc generates invalid Code in the default constructor for structs, when --cpp-field-case-style is used #7209 (#7211)
c9651b742 Add overloads for C# ByteBuffer/FlatBufferBuilder to allow adding vector blocks from ArraySegments or IntPtr (#7193)
26c3b3ada Update codeql.yml
da6e1b985 Update codeql.yml
ad27d751e Added Oss fuzz badge
0aab623cb Create codeql.yml
6a446bdd8 maximize parallel builds in CI (#7206)
21fb5cbbc Create scorecards.yml
0da6f9486 [C++] Static assert on Flatbuffers Version (#7203)
59e971308 reduce fuzzing time to 1 minute in CI
40866a892 fixed padding in struct for annotated binary (#7199)
b71d968fa Apply Namer prefix/suffix to other generators (#7197)
fac0d7be0 Apply Namer to Java. (#7194)
6c5603fd9 [C#] Fix collision of field name and type name (#7149)
2d21853a7 monster fuzzer fix for json default scalars
fec1a8d01 [swift] Add bazel configuration for Swift (#7195)
7fd857623 structured comments (#7192)
a4cb1599d Namerdart (#7187)
ae4ce7265 fuzzed binary annotator (#7188)
e2be0c0b0 Handle root offset and root table vtable invalidation (#7177)
2ad408697 [TS] Fix generation of struct members in object api (#7148)
4213d9105 VerifySizePrefixed (reflection::Schema) and GetAnySizePrefixedRoot added (#7181)
5a13f622c Correctly parse lists of enums in Dart generated code (#7157)
23a7e4e0b Adds no-includes flags to the swift code generator (#7182)
eeb49c275 Move flatbuffer_ts_library to typescript.bzl (#7183)
824763b31 Typo in flatc options (warning-as-errors instead of warnings-as-errors) (#7180)
d3aeee32b Annotated Flatbuffer Binary (#7174)
0bceba24d [Lua] Apply Namer to Lua (#7171)
b8c77d404 Make inclusion of header <optional> opt-out via macro (#7168)
8468eab83 Namersw (#7167)
2b2e8d4ae Nameroverloads (#7164)
b80b32bfa Use DESCRIPTION only if CMake version >= 3.9 (#7166)
Change-Id: Ic2681dabb1a798b7515e62753ee06aecb9933260
git-subtree-dir: third_party/flatbuffers
git-subtree-split: bc44fad35271e43fd7a79b4d691ac9e41708797f
Signed-off-by: Austin Schuh <austin.schuh@bluerivertech.com>
diff --git a/src/flatc.cpp b/src/flatc.cpp
index 3b2ef9f..1d12a17 100644
--- a/src/flatc.cpp
+++ b/src/flatc.cpp
@@ -19,11 +19,13 @@
#include <list>
#include <sstream>
+#include "annotated_binary_text_gen.h"
+#include "binary_annotator.h"
#include "flatbuffers/util.h"
namespace flatbuffers {
-const char *FLATC_VERSION() { return FLATBUFFERS_VERSION(); }
+static const char *FLATC_VERSION() { return FLATBUFFERS_VERSION(); }
void FlatCompiler::ParseFile(
flatbuffers::Parser &parser, const std::string &filename,
@@ -84,6 +86,8 @@
{ "", "scoped-enums", "",
"Use C++11 style scoped and strongly typed enums. Also implies "
"--no-prefix." },
+ { "", "swift-implementation-only", "",
+ "Adds a @_implementationOnly to swift imports" },
{ "", "gen-inclues", "",
"(deprecated), this is the default behavior. If the original behavior is "
"required (no include statements) use --no-includes." },
@@ -203,7 +207,7 @@
"Used with \"binary\" and \"json\" options, it generates data using "
"schema-less FlexBuffers." },
{ "", "no-warnings", "", "Inhibit all warnings messages." },
- { "", "warning-as-errors", "", "Treat all warnings as errors." },
+ { "", "warnings-as-errors", "", "Treat all warnings as errors." },
{ "", "cs-global-alias", "",
"Prepend \"global::\" to all user generated csharp classes and "
"structs." },
@@ -215,6 +219,11 @@
"in JSON, which is unsafe unless checked by a verifier afterwards." },
{ "", "ts-flat-files", "",
"Only generated one typescript file per .fbs file." },
+ { "", "annotate", "SCHEMA",
+ "Annotate the provided BINARY_FILE with the specified SCHEMA file." },
+ { "", "no-leak-private-annotation", "",
+ "Prevents multiple type of annotations within a Fbs SCHEMA file."
+ "Currently this is required to generate private types in Rust" },
};
static void AppendTextWrappedString(std::stringstream &ss, std::string &text,
@@ -297,7 +306,7 @@
ss << ", ";
}
ss.seekp(-2, ss.cur);
- ss << "]... FILE... [-- FILE...]";
+ ss << "]... FILE... [-- BINARY_FILE...]";
std::string help = ss.str();
std::stringstream ss_textwrap;
AppendTextWrappedString(ss_textwrap, help, 80, 0);
@@ -306,7 +315,8 @@
std::string FlatCompiler::GetUsageString(const char *program_name) const {
std::stringstream ss;
- ss << "Usage: " << program_name << " [OPTION]... FILE... [-- FILE...]\n";
+ ss << "Usage: " << program_name
+ << " [OPTION]... FILE... [-- BINARY_FILE...]\n";
for (size_t i = 0; i < params_.num_generators; ++i) {
const Generator &g = params_.generators[i];
AppendOption(ss, g.option, 80, 25);
@@ -320,16 +330,48 @@
std::string files_description =
"FILEs may be schemas (must end in .fbs), binary schemas (must end in "
- ".bfbs) or JSON files (conforming to preceding schema). FILEs after the "
- "-- must be binary flatbuffer format files. Output files are named using "
- "the base file name of the input, and written to the current directory "
- "or the path given by -o. example: " +
+ ".bfbs) or JSON files (conforming to preceding schema). BINARY_FILEs "
+ "after the -- must be binary flatbuffer format files. Output files are "
+ "named using the base file name of the input, and written to the current "
+ "directory or the path given by -o. example: " +
std::string(program_name) + " -c -b schema1.fbs schema2.fbs data.json";
AppendTextWrappedString(ss, files_description, 80, 0);
ss << "\n";
return ss.str();
}
+void FlatCompiler::AnnotateBinaries(
+ const uint8_t *binary_schema, const uint64_t binary_schema_size,
+ const std::string &schema_filename,
+ const std::vector<std::string> &binary_files) {
+ for (const std::string &filename : binary_files) {
+ std::string binary_contents;
+ if (!flatbuffers::LoadFile(filename.c_str(), true, &binary_contents)) {
+ Warn("unable to load binary file: " + filename);
+ continue;
+ }
+
+ const uint8_t *binary =
+ reinterpret_cast<const uint8_t *>(binary_contents.c_str());
+ const size_t binary_size = binary_contents.size();
+
+ flatbuffers::BinaryAnnotator binary_annotator(
+ binary_schema, binary_schema_size, binary, binary_size);
+
+ auto annotations = binary_annotator.Annotate();
+
+ // TODO(dbaileychess): Right now we just support a single text-based
+ // output of the annotated binary schema, which we generate here. We
+ // could output the raw annotations instead and have third-party tools
+ // use them to generate their own output.
+ flatbuffers::AnnotatedBinaryTextGenerator text_generator(
+ flatbuffers::AnnotatedBinaryTextGenerator::Options{}, annotations,
+ binary, binary_size);
+
+ text_generator.Generate(filename, schema_filename);
+ }
+}
+
int FlatCompiler::Compile(int argc, const char **argv) {
if (params_.generators == nullptr || params_.num_generators == 0) {
return 0;
@@ -353,6 +395,7 @@
std::vector<bool> generator_enabled(params_.num_generators, false);
size_t binary_files_from = std::numeric_limits<size_t>::max();
std::string conform_to_schema;
+ std::string annotate_schema;
const char *program_name = argv[0];
@@ -390,7 +433,7 @@
opts.include_prefix = flatbuffers::ConCatPathFileName(
flatbuffers::PosixPath(argv[argi]), "");
} else if (arg == "--keep-prefix") {
- opts.keep_include_path = true;
+ opts.keep_prefix = true;
} else if (arg == "--strict-json") {
opts.strict_json = true;
} else if (arg == "--allow-non-utf8") {
@@ -452,6 +495,8 @@
opts.java_checkerframework = true;
} else if (arg == "--gen-generated") {
opts.gen_generated = true;
+ } else if (arg == "--swift-implementation-only") {
+ opts.swift_implementation_only = true;
} else if (arg == "--gen-json-emit") {
opts.gen_json_coders = true;
} else if (arg == "--object-prefix") {
@@ -554,6 +599,11 @@
opts.json_nested_legacy_flatbuffers = true;
} else if (arg == "--ts-flat-files") {
opts.ts_flat_file = true;
+ } else if (arg == "--no-leak-private-annotation") {
+ opts.no_leak_private_annotations = true;
+ } else if (arg == "--annotate") {
+ if (++argi >= argc) Error("missing path following: " + arg, true);
+ annotate_schema = flatbuffers::PosixPath(argv[argi]);
} else {
for (size_t i = 0; i < params_.num_generators; ++i) {
if (arg == "--" + params_.generators[i].option.long_opt ||
@@ -582,7 +632,8 @@
if (opts.proto_mode) {
if (any_generator)
Error("cannot generate code directly from .proto files", true);
- } else if (!any_generator && conform_to_schema.empty()) {
+ } else if (!any_generator && conform_to_schema.empty() &&
+ annotate_schema.empty()) {
Error("no options: specify at least one generator.", true);
}
@@ -592,10 +643,6 @@
"well.");
}
- if (opts.ts_flat_file && opts.generate_all) {
- Error("Combining --ts-flat-file and --gen-all is not supported.");
- }
-
flatbuffers::Parser conform_parser;
if (!conform_to_schema.empty()) {
std::string contents;
@@ -611,6 +658,53 @@
}
}
+ if (!annotate_schema.empty()) {
+ const std::string ext = flatbuffers::GetExtension(annotate_schema);
+ if (!(ext == reflection::SchemaExtension() || ext == "fbs")) {
+ Error("Expected a `.bfbs` or `.fbs` schema, got: " + annotate_schema);
+ }
+
+ const bool is_binary_schema = ext == reflection::SchemaExtension();
+
+ std::string schema_contents;
+ if (!flatbuffers::LoadFile(annotate_schema.c_str(),
+ /*binary=*/is_binary_schema, &schema_contents)) {
+ Error("unable to load schema: " + annotate_schema);
+ }
+
+ const uint8_t *binary_schema = nullptr;
+ uint64_t binary_schema_size = 0;
+
+ IDLOptions binary_opts;
+ binary_opts.lang_to_generate |= flatbuffers::IDLOptions::kBinary;
+ flatbuffers::Parser parser(binary_opts);
+
+ if (is_binary_schema) {
+ binary_schema =
+ reinterpret_cast<const uint8_t *>(schema_contents.c_str());
+ binary_schema_size = schema_contents.size();
+ } else {
+ // If we need to generate the .bfbs file from the provided schema file
+ // (.fbs)
+ ParseFile(parser, annotate_schema, schema_contents, include_directories);
+ parser.Serialize();
+
+ binary_schema = parser.builder_.GetBufferPointer();
+ binary_schema_size = parser.builder_.GetSize();
+ }
+
+ if (binary_schema == nullptr || !binary_schema_size) {
+ Error("could not parse a value binary schema from: " + annotate_schema);
+ }
+
+ // Annotate the provided files with the binary_schema.
+ AnnotateBinaries(binary_schema, binary_schema_size, annotate_schema,
+ filenames);
+
+ // We don't support doing anything else after annotating a binary.
+ return 0;
+ }
+
std::unique_ptr<flatbuffers::Parser> parser(new flatbuffers::Parser(opts));
for (auto file_it = filenames.begin(); file_it != filenames.end();
@@ -755,7 +849,7 @@
if (params_.generators[i].generateGRPC != nullptr) {
if (!params_.generators[i].generateGRPC(*parser.get(), output_path,
filebase)) {
- Error(std::string("Unable to generate GRPC interface for") +
+ Error(std::string("Unable to generate GRPC interface for ") +
params_.generators[i].lang_name);
}
} else {