Add a "configuration" channel to MCAP files for foxglove
This is helpful for doing certain types of analysis where you might want
to, e.g., correlate channel indices to a channel name/type (e.g., for
analyzing sent-too-fast errors or for using the ReplayTiming
message implemented by I471fefd96a4d043766b54dd4488726e24926a95f).
Change-Id: Ic68026be58205607a2099ccfd7547187989ec26c
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/aos/BUILD b/aos/BUILD
index b7aef88..a1c92e7 100644
--- a/aos/BUILD
+++ b/aos/BUILD
@@ -1,5 +1,6 @@
load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library", "flatbuffer_py_library", "flatbuffer_rust_library", "flatbuffer_ts_library")
load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
+load("//aos:flatbuffers.bzl", "cc_static_flatbuffer")
load("//tools/build_rules:autocxx.bzl", "autocxx_library")
exports_files(["aos_dump_autocomplete.sh"])
@@ -201,6 +202,13 @@
visibility = ["//visibility:public"],
)
+cc_static_flatbuffer(
+ name = "configuration_schema",
+ function = "aos::ConfigurationSchema",
+ target = ":configuration_fbs_reflection_out",
+ visibility = ["//visibility:public"],
+)
+
flatbuffer_ts_library(
name = "configuration_ts_fbs",
srcs = ["configuration.fbs"],
diff --git a/aos/util/BUILD b/aos/util/BUILD
index 12e5ab7..faa3dc1 100644
--- a/aos/util/BUILD
+++ b/aos/util/BUILD
@@ -54,6 +54,7 @@
target_compatible_with = ["@platforms//os:linux"],
deps = [
"//aos:configuration_fbs",
+ "//aos:configuration_schema",
"//aos:fast_string_builder",
"//aos:flatbuffer_utils",
"//aos/events:event_loop",
diff --git a/aos/util/foxglove-layouts/README b/aos/util/foxglove-layouts/README
new file mode 100644
index 0000000..7ec6341
--- /dev/null
+++ b/aos/util/foxglove-layouts/README
@@ -0,0 +1,4 @@
+This folder contains JSON layouts that may be helpful for analyzing AOS-generic problems.
+
+Currently, this includes:
+* "Sent-Too-Fast Analysis": A sample layout for analyzing sent-too-fast errors.
diff --git a/aos/util/foxglove-layouts/Sent-Too-Fast Analysis.json b/aos/util/foxglove-layouts/Sent-Too-Fast Analysis.json
new file mode 100644
index 0000000..f558cc5
--- /dev/null
+++ b/aos/util/foxglove-layouts/Sent-Too-Fast Analysis.json
@@ -0,0 +1,88 @@
+{
+ "configById": {
+ "NodePlayground!3jd6ze5": {
+ "selectedNodeId": "a2a921fe-ea55-45ed-881f-d66ad46bc666",
+ "autoFormatOnSave": true
+ },
+ "RawMessages!qt4g88": {
+ "topicPath": "configuration",
+ "diffTopicPath": "",
+ "diffMethod": "custom",
+ "diffEnabled": false,
+ "showFullMessageForDiff": false,
+ "autoExpandMode": "auto"
+ },
+ "RawMessages!29a8tmz": {
+ "topicPath": "\"/aos aos.timing.Report\"{name==\"aos_send\"}",
+ "diffTopicPath": "",
+ "diffMethod": "custom",
+ "diffEnabled": false,
+ "showFullMessageForDiff": false,
+ "autoExpandMode": "auto"
+ },
+ "RawMessages!26t8w4x": {
+ "topicPath": "/send_errors.channels[:]{type==\"aos.logging.LogMessageFbs\"}{name==\"/aos\"}",
+ "diffTopicPath": "",
+ "diffMethod": "custom",
+ "diffEnabled": false,
+ "showFullMessageForDiff": false,
+ "autoExpandMode": "auto"
+ },
+ "SourceInfo!4j44duu": {},
+ "Plot!26dzl0q": {
+ "title": "Plot",
+ "paths": [
+ {
+ "value": "\"/aos aos.timing.Report\".senders[:].error_counts[0].count",
+ "enabled": true,
+ "timestampMethod": "receiveTime"
+ }
+ ],
+ "showXAxisLabels": true,
+ "showYAxisLabels": true,
+ "showLegend": true,
+ "legendDisplay": "floating",
+ "showPlotValuesInLegend": false,
+ "isSynced": true,
+ "xAxisVal": "timestamp",
+ "sidebarDimension": 240
+ }
+ },
+ "globalVariables": {},
+ "userNodes": {
+ "a2a921fe-ea55-45ed-881f-d66ad46bc666": {
+ "sourceCode": "// The ./types module provides helper types for your Input events and messages.\nimport { Input, Message } from \"./types\";\n\n// Your node can output well-known message types, any of your custom message types, or\n// complete custom message types.\n//\n// Use `Message` to access your data source types or well-known types:\n// type Twist = Message<\"geometry_msgs/Twist\">;\n//\n// Conventionally, it's common to make a _type alias_ for your node's output type\n// and use that type name as the return type for your node function.\n// Here we've called the type `Output` but you can pick any type name.\ntype ChannelSendErrors = {\n name: string;\n type: string;\n sent_too_fast_errors: number;\n all_send_errors: number;\n};\ntype AggregateSendErrors = {\n channels: ChannelSendErrors[];\n};\n\n// These are the topics your node \"subscribes\" to. Studio will invoke your node function\n// when any message is received on one of these topics.\nexport const inputs = [\"configuration\", \"/aos aos.timing.Report\"];\n\n// Any output your node produces is \"published\" to this topic. Published messages are only visible within Studio, not to your original data source.\nexport const output = \"/send_errors\";\n\nlet config: any = undefined;\n\nconst errors: AggregateSendErrors = { channels: [] };\n\n// This function is called with messages from your input topics.\n// The first argument is an event with the topic, receive time, and message.\n// Use the `Input<...>` helper to get the correct event type for your input topic messages.\nexport default function node(\n event: Input<\"configuration\"> | Input<\"/aos aos.timing.Report\">\n): AggregateSendErrors | undefined {\n if (event.topic == \"configuration\") {\n config = event.message;\n for (const channel of event.message.channels) {\n errors.channels.push({\n name: channel.name,\n type: channel.type,\n sent_too_fast_errors: 0,\n all_send_errors: 0,\n });\n }\n } else {\n if (config == undefined) {\n return;\n }\n for (const channel_errors of errors.channels) {\n for (const sender of event.message.senders) {\n const channel = config.channels[sender.channel_index];\n if (\n channel.name == channel_errors.name &&\n channel.type == channel_errors.type\n ) {\n for (const error_count of sender.error_counts) {\n if (error_count.error == 0) {\n channel_errors.sent_too_fast_errors += error_count.count;\n }\n channel_errors.all_send_errors += error_count.count;\n }\n return errors;\n }\n }\n }\n }\n return;\n}\n",
+ "name": "a2a921fe"
+ }
+ },
+ "linkedGlobalVariables": [],
+ "playbackConfig": {
+ "speed": 5,
+ "messageOrder": "receiveTime"
+ },
+ "layout": {
+ "first": {
+ "first": "NodePlayground!3jd6ze5",
+ "second": "Plot!26dzl0q",
+ "direction": "row"
+ },
+ "second": {
+ "first": "RawMessages!qt4g88",
+ "second": {
+ "first": {
+ "first": "RawMessages!29a8tmz",
+ "second": "RawMessages!26t8w4x",
+ "direction": "column",
+ "splitPercentage": 46.354586297778205
+ },
+ "second": "SourceInfo!4j44duu",
+ "direction": "column",
+ "splitPercentage": 67.88193961641498
+ },
+ "direction": "column",
+ "splitPercentage": 36.885245901639344
+ },
+ "direction": "row",
+ "splitPercentage": 70
+ }
+}
diff --git a/aos/util/mcap_logger.cc b/aos/util/mcap_logger.cc
index 561a02b..dc27504 100644
--- a/aos/util/mcap_logger.cc
+++ b/aos/util/mcap_logger.cc
@@ -1,6 +1,7 @@
#include "aos/util/mcap_logger.h"
#include "absl/strings/str_replace.h"
+#include "aos/configuration_schema.h"
#include "aos/flatbuffer_merge.h"
#include "single_include/nlohmann/json.hpp"
@@ -85,7 +86,29 @@
Serialization serialization)
: event_loop_(event_loop),
output_(output_path),
- serialization_(serialization) {
+ serialization_(serialization),
+ configuration_channel_([]() {
+ // Setup a fake Channel for providing the configuration in the MCAP
+ // file. This is included for convenience so that consumers of the MCAP
+ // file can actually dereference things like the channel indices in AOS
+ // timing reports.
+ flatbuffers::FlatBufferBuilder fbb;
+ flatbuffers::Offset<flatbuffers::String> name_offset =
+ fbb.CreateString("");
+ flatbuffers::Offset<flatbuffers::String> type_offset =
+ fbb.CreateString("aos.Configuration");
+ flatbuffers::Offset<reflection::Schema> schema_offset =
+ aos::CopyFlatBuffer(
+ aos::FlatbufferSpan<reflection::Schema>(ConfigurationSchema()),
+ &fbb);
+ Channel::Builder channel(fbb);
+ channel.add_name(name_offset);
+ channel.add_type(type_offset);
+ channel.add_schema(schema_offset);
+ fbb.Finish(channel.Finish());
+ return fbb.Release();
+ }()),
+ configuration_(CopyFlatBuffer(event_loop_->configuration())) {
event_loop->SkipTimingReport();
event_loop->SkipAosLog();
CHECK(output_);
@@ -181,6 +204,20 @@
}
}
+ // Manually add in a special /configuration channel.
+ if (register_handlers == RegisterHandlers::kYes) {
+ configuration_id_ = ++id;
+ event_loop_->OnRun([this]() {
+ Context config_context;
+ config_context.monotonic_event_time = event_loop_->monotonic_now();
+ config_context.queue_index = 0;
+ config_context.size = configuration_.span().size();
+ config_context.data = configuration_.span().data();
+ WriteMessage(configuration_id_, &configuration_channel_.message(),
+ config_context, ¤t_chunk_);
+ });
+ }
+
std::vector<SummaryOffset> offsets;
const uint64_t schema_offset = output_.tellp();
@@ -189,6 +226,8 @@
WriteSchema(pair.first, pair.second);
}
+ WriteSchema(configuration_id_, &configuration_channel_.message());
+
const uint64_t channel_offset = output_.tellp();
offsets.push_back(
@@ -201,6 +240,13 @@
WriteChannel(pair.first, pair.first, pair.second);
}
+ // Provide the configuration message on a special channel that is just named
+ // "configuration", which is guaranteed not to conflict with existing under
+ // our current naming scheme (since our current scheme will, at a minimum, put
+ // a space between the name/type of a channel).
+ WriteChannel(configuration_id_, configuration_id_,
+ &configuration_channel_.message(), "configuration");
+
offsets.push_back({OpCode::kChannel, channel_offset,
static_cast<uint64_t>(output_.tellp()) - channel_offset});
return offsets;
@@ -267,7 +313,8 @@
}
void McapLogger::WriteChannel(const uint16_t id, const uint16_t schema_id,
- const aos::Channel *channel) {
+ const aos::Channel *channel,
+ std::string_view override_name) {
string_builder_.Reset();
// Channel ID
AppendInt16(&string_builder_, id);
@@ -275,8 +322,10 @@
AppendInt16(&string_builder_, schema_id);
// Topic name
AppendString(&string_builder_,
- absl::StrCat(channel->name()->string_view(), " ",
- channel->type()->string_view()));
+ override_name.empty()
+ ? absl::StrCat(channel->name()->string_view(), " ",
+ channel->type()->string_view())
+ : override_name);
// Encoding
switch (serialization_) {
case Serialization::kJson:
diff --git a/aos/util/mcap_logger.h b/aos/util/mcap_logger.h
index 5ae6413..d7409fb 100644
--- a/aos/util/mcap_logger.h
+++ b/aos/util/mcap_logger.h
@@ -95,7 +95,8 @@
void WriteDataEnd();
void WriteSchema(const uint16_t id, const aos::Channel *channel);
void WriteChannel(const uint16_t id, const uint16_t schema_id,
- const aos::Channel *channel);
+ const aos::Channel *channel,
+ std::string_view override_name = "");
void WriteMessage(uint16_t channel_id, const Channel *channel,
const Context &context, std::ostream *output);
void WriteChunk();
@@ -154,6 +155,13 @@
message_indices_;
// ChunkIndex's for all fully written Chunks.
std::vector<ChunkIndex> chunk_indices_;
+
+ // Metadata associated with the fake "configuration" channel that we create in
+ // order to ensure that foxglove extensions/users have access to the full
+ // configuration.
+ uint16_t configuration_id_ = 0;
+ FlatbufferDetachedBuffer<Channel> configuration_channel_;
+ FlatbufferDetachedBuffer<Configuration> configuration_;
};
} // namespace aos
#endif // AOS_UTIL_MCAP_LOGGER_H_