Add support for triming vectors in json
Sometimes (we have images we want to print) we don't want to print out
all the data from long arrays in JSON. That can be too slow.
Add an option which lets us limit the number of elements to print before
printing out a "... xxx elements ..." message instead.
Change-Id: I95ad6ea3e9e2fe1767005be8f53c8ba8d4371ace
diff --git a/aos/aos_dump.cc b/aos/aos_dump.cc
index 2ecfe5e..fd33eca 100644
--- a/aos/aos_dump.cc
+++ b/aos/aos_dump.cc
@@ -8,6 +8,8 @@
#include "gflags/gflags.h"
DEFINE_string(config, "./config.json", "File path of aos configuration");
+DEFINE_int32(max_vector_size, 100,
+ "If positive, vectors longer than this will not be printed");
int main(int argc, char **argv) {
aos::InitGoogle(&argc, &argv);
@@ -57,14 +59,16 @@
<< context.monotonic_event_time << "): "
<< aos::FlatbufferToJson(
channel->schema(),
- static_cast<const uint8_t *>(message))
+ static_cast<const uint8_t *>(message),
+ FLAGS_max_vector_size)
<< '\n';
} else {
std::cout << context.realtime_event_time << " ("
<< context.monotonic_event_time << "): "
<< aos::FlatbufferToJson(
channel->schema(),
- static_cast<const uint8_t *>(message))
+ static_cast<const uint8_t *>(message),
+ FLAGS_max_vector_size)
<< '\n';
}
});
diff --git a/aos/flatbuffer_introspection.cc b/aos/flatbuffer_introspection.cc
index 751666d..036a830 100644
--- a/aos/flatbuffer_introspection.cc
+++ b/aos/flatbuffer_introspection.cc
@@ -1,3 +1,4 @@
+#include <cmath>
#include <iostream>
#include <sstream>
@@ -46,6 +47,10 @@
void FloatToString(double val, reflection::BaseType type,
std::stringstream *out) {
+ if (std::isnan(val)) {
+ *out << "null";
+ return;
+ }
switch (type) {
case BaseType::Float:
out->precision(std::numeric_limits<float>::digits10);
@@ -65,7 +70,7 @@
const reflection::Object *obj,
const flatbuffers::Vector<flatbuffers::Offset<reflection::Object>> *objects,
const flatbuffers::Vector<flatbuffers::Offset<reflection::Enum>> *enums,
- const ObjT *object, std::stringstream *out);
+ const ObjT *object, size_t max_vector_size, std::stringstream *out);
// Get enum value name
const char *EnumToString(
@@ -95,6 +100,8 @@
if (value_string != nullptr) {
*out << '"' << value_string << '"';
+ } else {
+ *out << val;
}
}
} else {
@@ -113,7 +120,7 @@
const ObjT *table, const reflection::Field *field,
const flatbuffers::Vector<flatbuffers::Offset<reflection::Object>> *objects,
const flatbuffers::Vector<flatbuffers::Offset<reflection::Enum>> *enums,
- std::stringstream *out) {
+ size_t max_vector_size, std::stringstream *out) {
const reflection::Type *type = field->type();
switch (type->base_type()) {
@@ -176,6 +183,10 @@
flatbuffers::GetFieldAnyV(*table, *field);
reflection::BaseType elem_type = type->element();
+ if (vector->size() > max_vector_size) {
+ *out << "[ ... " << vector->size() << " elements ... ]";
+ break;
+ }
*out << '[';
for (flatbuffers::uoffset_t i = 0; i < vector->size(); ++i) {
if (i != 0) {
@@ -200,11 +211,13 @@
flatbuffers::GetAnyVectorElemAddressOf<
const flatbuffers::Struct>(
vector, i, objects->Get(type->index())->bytesize()),
+ max_vector_size,
out);
} else {
ObjectToString(objects->Get(type->index()), objects, enums,
flatbuffers::GetAnyVectorElemPointer<
const flatbuffers::Table>(vector, i),
+ max_vector_size,
out);
}
}
@@ -219,10 +232,10 @@
if (type->index() > -1 && type->index() < (int32_t)objects->size()) {
if (objects->Get(type->index())->is_struct()) {
ObjectToString(objects->Get(type->index()), objects, enums,
- flatbuffers::GetFieldStruct(*table, *field), out);
+ flatbuffers::GetFieldStruct(*table, *field), max_vector_size, out);
} else if constexpr (std::is_same<flatbuffers::Table, ObjT>()) {
ObjectToString(objects->Get(type->index()), objects, enums,
- flatbuffers::GetFieldT(*table, *field), out);
+ flatbuffers::GetFieldT(*table, *field), max_vector_size,out);
}
} else {
*out << "null";
@@ -240,7 +253,7 @@
const reflection::Object *obj,
const flatbuffers::Vector<flatbuffers::Offset<reflection::Object>> *objects,
const flatbuffers::Vector<flatbuffers::Offset<reflection::Enum>> *enums,
- const ObjT *object, std::stringstream *out) {
+ const ObjT *object, size_t max_vector_size, std::stringstream *out) {
static_assert(std::is_same<flatbuffers::Table, ObjT>() ||
std::is_same<flatbuffers::Struct, ObjT>(),
"Type must be either flatbuffer table or struct");
@@ -256,7 +269,7 @@
print_sep = true;
}
*out << '"' << field->name()->c_str() << "\": ";
- FieldToString(object, field, objects, enums, out);
+ FieldToString(object, field, objects, enums, max_vector_size, out);
}
}
*out << '}';
@@ -265,14 +278,15 @@
} // namespace
std::string FlatbufferToJson(const reflection::Schema *schema,
- const uint8_t *data) {
+ const uint8_t *data, size_t max_vector_size) {
const flatbuffers::Table *table = flatbuffers::GetAnyRoot(data);
const reflection::Object *obj = schema->root_table();
std::stringstream out;
- ObjectToString(obj, schema->objects(), schema->enums(), table, &out);
+ ObjectToString(obj, schema->objects(), schema->enums(), table,
+ max_vector_size, &out);
return out.str();
}
diff --git a/aos/flatbuffer_introspection_test.cc b/aos/flatbuffer_introspection_test.cc
index 893b402..9da705f 100644
--- a/aos/flatbuffer_introspection_test.cc
+++ b/aos/flatbuffer_introspection_test.cc
@@ -314,5 +314,23 @@
EXPECT_EQ(out, "{\"foo_string\": \"\\\"\\\\\\b\\f\\n\\r\\t\"}");
}
+TEST_F(FlatbufferIntrospectionTest, TrimmedVector) {
+ flatbuffers::FlatBufferBuilder builder;
+
+ std::vector<int32_t> contents;
+ for (int i = 0; i < 101; ++i) {
+ contents.push_back(i);
+ }
+ const auto contents_offset = builder.CreateVector(contents);
+
+ ConfigurationBuilder config_builder(builder);
+ config_builder.add_vector_foo_int(contents_offset);
+
+ builder.Finish(config_builder.Finish());
+
+ std::string out = FlatbufferToJson(schema_, builder.GetBufferPointer(), 100);
+ EXPECT_EQ(out, "{\"vector_foo_int\": [ ... 101 elements ... ]}");
+}
+
} // namespace testing
} // namespace aos
diff --git a/aos/json_to_flatbuffer.cc b/aos/json_to_flatbuffer.cc
index 95c95dd..fdab330 100644
--- a/aos/json_to_flatbuffer.cc
+++ b/aos/json_to_flatbuffer.cc
@@ -90,7 +90,6 @@
flatbuffers::Offset<flatbuffers::String> offset_element,
flatbuffers::FlatBufferBuilder *fbb);
-
// Writes an array of FieldElement (with the definition in the type
// table) to the builder. Returns the offset of the table.
flatbuffers::uoffset_t WriteTable(const flatbuffers::TypeTable *typetable,
@@ -146,8 +145,7 @@
// Parses the flatbuffer. This is a second method so we can do easier
// cleanup at the top level. Returns true on success.
bool DoParse(const flatbuffers::TypeTable *typetable,
- const std::string_view data,
- flatbuffers::uoffset_t *table_end);
+ const std::string_view data, flatbuffers::uoffset_t *table_end);
// Adds *_value for the provided field. If we are in a vector, queues the
// data up in vector_elements. Returns true on success.
@@ -249,8 +247,8 @@
return false;
} else {
// End of a nested struct! Add it.
- const flatbuffers::uoffset_t end = WriteTable(
- stack_.back().typetable, stack_.back().elements, fbb_);
+ const flatbuffers::uoffset_t end =
+ WriteTable(stack_.back().typetable, stack_.back().elements, fbb_);
// We now want to talk about the parent structure. Pop the child.
stack_.pop_back();
@@ -774,8 +772,7 @@
}
flatbuffers::DetachedBuffer JsonToFlatbuffer(
- const std::string_view data,
- const flatbuffers::TypeTable *typetable) {
+ const std::string_view data, const flatbuffers::TypeTable *typetable) {
flatbuffers::FlatBufferBuilder fbb;
fbb.ForceDefaults(true);
@@ -793,7 +790,7 @@
::std::string BufferFlatbufferToJson(const uint8_t *buffer,
const ::flatbuffers::TypeTable *typetable,
- bool multi_line) {
+ bool multi_line, size_t max_vector_size) {
// It is pretty common to get passed in a nullptr when a test fails. Rather
// than CHECK, return a more user friendly result.
if (buffer == nullptr) {
@@ -801,22 +798,145 @@
}
return TableFlatbufferToJson(reinterpret_cast<const flatbuffers::Table *>(
flatbuffers::GetRoot<uint8_t>(buffer)),
- typetable, multi_line);
+ typetable, multi_line, max_vector_size);
}
+namespace {
+
+// A visitor which manages skipping the contents of vectors that are longer than
+// a specified threshold.
+class TruncatingStringVisitor : public flatbuffers::IterationVisitor {
+ public:
+ TruncatingStringVisitor(size_t max_vector_size, std::string delimiter,
+ bool quotes, std::string indent, bool vdelimited)
+ : max_vector_size_(max_vector_size),
+ to_string_(delimiter, quotes, indent, vdelimited) {}
+ ~TruncatingStringVisitor() override {}
+
+ void StartSequence() override {
+ if (should_skip()) return;
+ to_string_.StartSequence();
+ }
+ void EndSequence() override {
+ if (should_skip()) return;
+ to_string_.EndSequence();
+ }
+ void Field(size_t field_idx, size_t set_idx, flatbuffers::ElementaryType type,
+ bool is_vector, const flatbuffers::TypeTable *type_table,
+ const char *name, const uint8_t *val) override {
+ if (should_skip()) return;
+ to_string_.Field(field_idx, set_idx, type, is_vector, type_table, name,
+ val);
+ }
+ void UType(uint8_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.UType(value, name);
+ }
+ void Bool(bool value) override {
+ if (should_skip()) return;
+ to_string_.Bool(value);
+ }
+ void Char(int8_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.Char(value, name);
+ }
+ void UChar(uint8_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.UChar(value, name);
+ }
+ void Short(int16_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.Short(value, name);
+ }
+ void UShort(uint16_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.UShort(value, name);
+ }
+ void Int(int32_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.Int(value, name);
+ }
+ void UInt(uint32_t value, const char *name) override {
+ if (should_skip()) return;
+ to_string_.UInt(value, name);
+ }
+ void Long(int64_t value) override {
+ if (should_skip()) return;
+ to_string_.Long(value);
+ }
+ void ULong(uint64_t value) override {
+ if (should_skip()) return;
+ to_string_.ULong(value);
+ }
+ void Float(float value) override {
+ if (should_skip()) return;
+ to_string_.Float(value);
+ }
+ void Double(double value) override {
+ if (should_skip()) return;
+ to_string_.Double(value);
+ }
+ void String(const flatbuffers::String *value) override {
+ if (should_skip()) return;
+ to_string_.String(value);
+ }
+ void Unknown(const uint8_t *value) override {
+ if (should_skip()) return;
+ to_string_.Unknown(value);
+ }
+ void Element(size_t i, flatbuffers::ElementaryType type,
+ const flatbuffers::TypeTable *type_table,
+ const uint8_t *val) override {
+ if (should_skip()) return;
+ to_string_.Element(i, type, type_table, val);
+ }
+
+ virtual void StartVector(size_t size) override {
+ if (should_skip()) {
+ ++skip_levels_;
+ return;
+ }
+ if (size > max_vector_size_) {
+ ++skip_levels_;
+ to_string_.s += "[ ... " + std::to_string(size) + " elements ... ]";
+ return;
+ }
+ to_string_.StartVector(size);
+ }
+ virtual void EndVector() override {
+ if (should_skip()) {
+ --skip_levels_;
+ return;
+ }
+ to_string_.EndVector();
+ }
+
+ std::string &string() { return to_string_.s; }
+
+ private:
+ bool should_skip() const { return skip_levels_ > 0; }
+
+ const size_t max_vector_size_;
+ flatbuffers::ToStringVisitor to_string_;
+ int skip_levels_ = 0;
+};
+
+} // namespace
+
::std::string TableFlatbufferToJson(const flatbuffers::Table *t,
const ::flatbuffers::TypeTable *typetable,
- bool multi_line) {
+ bool multi_line, size_t max_vector_size) {
// It is pretty common to get passed in a nullptr when a test fails. Rather
// than CHECK, return a more user friendly result.
if (t == nullptr) {
return "null";
}
- ::flatbuffers::ToStringVisitor tostring_visitor(
- multi_line ? "\n" : " ", true, multi_line ? " " : "", multi_line);
+ TruncatingStringVisitor tostring_visitor(max_vector_size,
+ multi_line ? "\n" : " ", true,
+ multi_line ? " " : "", multi_line);
flatbuffers::IterateObject(reinterpret_cast<const uint8_t *>(t), typetable,
&tostring_visitor);
- return tostring_visitor.s;
+ return tostring_visitor.string();
}
} // namespace aos
diff --git a/aos/json_to_flatbuffer.h b/aos/json_to_flatbuffer.h
index c061c69..cadc0de 100644
--- a/aos/json_to_flatbuffer.h
+++ b/aos/json_to_flatbuffer.h
@@ -39,40 +39,46 @@
// TableFlatbufferToJson.
::std::string BufferFlatbufferToJson(const uint8_t *buffer,
const flatbuffers::TypeTable *typetable,
- bool multi_line = false);
+ bool multi_line = false,
+ size_t max_vector_size = SIZE_MAX);
::std::string TableFlatbufferToJson(const flatbuffers::Table *t,
const ::flatbuffers::TypeTable *typetable,
- bool multi_line);
+ bool multi_line, size_t max_vector_size = SIZE_MAX);
// Converts a DetachedBuffer holding a flatbuffer to JSON.
inline ::std::string FlatbufferToJson(const flatbuffers::DetachedBuffer &buffer,
const flatbuffers::TypeTable *typetable,
- bool multi_line = false) {
- return BufferFlatbufferToJson(buffer.data(), typetable, multi_line);
+ bool multi_line = false,
+ size_t max_vector_size = SIZE_MAX) {
+ return BufferFlatbufferToJson(buffer.data(), typetable, multi_line,
+ max_vector_size);
}
// Converts a Flatbuffer<T> holding a flatbuffer to JSON.
template <typename T>
inline ::std::string FlatbufferToJson(const Flatbuffer<T> &flatbuffer,
- bool multi_line = false) {
- return BufferFlatbufferToJson(
- flatbuffer.data(), Flatbuffer<T>::MiniReflectTypeTable(), multi_line);
+ bool multi_line = false,
+ size_t max_vector_size = SIZE_MAX) {
+ return BufferFlatbufferToJson(flatbuffer.data(),
+ Flatbuffer<T>::MiniReflectTypeTable(),
+ multi_line, max_vector_size);
}
// Converts a flatbuffer::Table to JSON.
template <typename T>
-typename std::enable_if<
- std::is_base_of<flatbuffers::Table, T>::value,
- std::string>::type inline FlatbufferToJson(const T *flatbuffer,
- bool multi_line = false) {
+typename std::enable_if<std::is_base_of<flatbuffers::Table, T>::value,
+ std::string>::
+ type inline FlatbufferToJson(const T *flatbuffer, bool multi_line = false,
+ size_t max_vector_size = SIZE_MAX) {
return TableFlatbufferToJson(
reinterpret_cast<const flatbuffers::Table *>(flatbuffer),
- Flatbuffer<T>::MiniReflectTypeTable(), multi_line);
+ Flatbuffer<T>::MiniReflectTypeTable(), multi_line, max_vector_size);
}
std::string FlatbufferToJson(const reflection::Schema *const schema,
- const uint8_t *const data);
+ const uint8_t *const data,
+ size_t max_vector_size = SIZE_MAX);
} // namespace aos
diff --git a/aos/json_to_flatbuffer_test.cc b/aos/json_to_flatbuffer_test.cc
index 6c76b02..a9112f4 100644
--- a/aos/json_to_flatbuffer_test.cc
+++ b/aos/json_to_flatbuffer_test.cc
@@ -205,5 +205,31 @@
//
// TODO(austin): unions?
+TEST_F(JsonToFlatbufferTest, TrimmedVector) {
+ std::string json_short = "{ \"vector_foo_int\": [ 0";
+ for (int i = 1; i < 100; ++i) {
+ json_short += ", ";
+ json_short += std::to_string(i);
+ }
+ std::string json_long = json_short;
+ json_short += " ] }";
+ json_long += ", 101 ] }";
+
+ const flatbuffers::DetachedBuffer fb_short =
+ JsonToFlatbuffer(json_short.data(), ConfigurationTypeTable());
+ ASSERT_GT(fb_short.size(), 0);
+ const flatbuffers::DetachedBuffer fb_long =
+ JsonToFlatbuffer(json_long.data(), ConfigurationTypeTable());
+ ASSERT_GT(fb_long.size(), 0);
+
+ const std::string back_json_short =
+ FlatbufferToJson(fb_short, ConfigurationTypeTable(), false, 100);
+ const std::string back_json_long =
+ FlatbufferToJson(fb_long, ConfigurationTypeTable(), false, 100);
+
+ EXPECT_EQ(json_short, back_json_short);
+ EXPECT_EQ("{ \"vector_foo_int\": [ ... 101 elements ... ] }", back_json_long);
+}
+
} // namespace testing
} // namespace aos
diff --git a/third_party/flatbuffers/include/flatbuffers/minireflect.h b/third_party/flatbuffers/include/flatbuffers/minireflect.h
index e7b9024..052c60e 100644
--- a/third_party/flatbuffers/include/flatbuffers/minireflect.h
+++ b/third_party/flatbuffers/include/flatbuffers/minireflect.h
@@ -61,7 +61,7 @@
virtual void String(const String *) {}
virtual void Unknown(const uint8_t *) {} // From a future version.
// These mark the scope of a vector.
- virtual void StartVector() {}
+ virtual void StartVector(size_t /*size*/) {}
virtual void EndVector() {}
virtual void Element(size_t /*i*/, ElementaryType /*type*/,
const TypeTable * /*type_table*/,
@@ -254,7 +254,7 @@
if (is_vector) {
val += ReadScalar<uoffset_t>(val);
auto vec = reinterpret_cast<const Vector<uint8_t> *>(val);
- visitor->StartVector();
+ visitor->StartVector(vec->size());
auto elem_ptr = vec->Data();
for (size_t j = 0; j < vec->size(); j++) {
visitor->Element(j, type, ref, elem_ptr);
@@ -358,7 +358,7 @@
EscapeString(str->c_str(), str->size(), &s, true, false);
}
void Unknown(const uint8_t *) { s += "(?)"; }
- void StartVector() {
+ void StartVector(size_t /*size*/) {
s += "[";
if (vector_delimited) {
s += d;