Track message_bridge client UUID and connection counts and time
It is hard to tell from the status message if a node has been
reconnecting a bunch or how long it has been connected. Add both of
those numbers to both the client and server statistics.
While we are here, also add the server's boot UUID to the client.
Because of the way the protocol works, we only get this when the first
message is received, but that happens pretty quickly due to the
timestamps.
All of this should give us much better debugging around client/server
connections when things start to go south.
Change-Id: I2ed732afc81a045c7701fe47d1460a7df9c3d778
Signed-off-by: Austin Schuh <austin.linux@gmail.com>
diff --git a/aos/events/logging/logger_test.cc b/aos/events/logging/logger_test.cc
index 97b293a..7a569eb 100644
--- a/aos/events/logging/logger_test.cc
+++ b/aos/events/logging/logger_test.cc
@@ -2614,9 +2614,9 @@
}
constexpr std::string_view kCombinedConfigSha1(
- "644a3ab079c3ddfb1ef866483cfca9ec015c4142202169081138f72664ffd589");
+ "9e07da76098ad1b755a7c3143aca300d66b6abb88745f6c36e603ef1441f0ad5");
constexpr std::string_view kSplitConfigSha1(
- "20bb9f6ee89519c4bd49986f0afe497d61c71b29d846f2c51f51727c9ef37415");
+ "85ef8be228bf4eb36f4d64ba68183b2a9a616bfb9b057e430d61e33bd273df86");
INSTANTIATE_TEST_SUITE_P(
All, MultinodeLoggerTest,
diff --git a/aos/events/simulated_event_loop_test.cc b/aos/events/simulated_event_loop_test.cc
index 71d4138..277e783 100644
--- a/aos/events/simulated_event_loop_test.cc
+++ b/aos/events/simulated_event_loop_test.cc
@@ -518,6 +518,8 @@
for (const message_bridge::ServerConnection *connection :
*stats.connections()) {
EXPECT_EQ(connection->state(), message_bridge::State::CONNECTED);
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_EQ(connection->connected_since_time(), 0);
EXPECT_TRUE(connection->has_boot_uuid());
if (connection->node()->name()->string_view() == "pi2") {
EXPECT_GT(connection->sent_packets(), 50);
@@ -547,6 +549,8 @@
EXPECT_GT(connection->sent_packets(), 50);
EXPECT_TRUE(connection->has_monotonic_offset());
EXPECT_EQ(connection->monotonic_offset(), 0);
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_EQ(connection->connected_since_time(), 0);
++pi2_server_statistics_count;
});
@@ -564,6 +568,8 @@
EXPECT_GE(connection->sent_packets(), 5);
EXPECT_TRUE(connection->has_monotonic_offset());
EXPECT_EQ(connection->monotonic_offset(), 0);
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_EQ(connection->connected_since_time(), 0);
++pi3_server_statistics_count;
});
@@ -588,6 +594,8 @@
EXPECT_EQ(connection->partial_deliveries(), 0);
EXPECT_TRUE(connection->has_monotonic_offset());
EXPECT_EQ(connection->monotonic_offset(), 150000);
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_EQ(connection->connected_since_time(), 0);
}
++pi1_client_statistics_count;
});
@@ -606,6 +614,8 @@
EXPECT_EQ(connection->partial_deliveries(), 0);
EXPECT_TRUE(connection->has_monotonic_offset());
EXPECT_EQ(connection->monotonic_offset(), 150000);
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_EQ(connection->connected_since_time(), 0);
++pi2_client_statistics_count;
});
@@ -623,6 +633,8 @@
EXPECT_EQ(connection->partial_deliveries(), 0);
EXPECT_TRUE(connection->has_monotonic_offset());
EXPECT_EQ(connection->monotonic_offset(), 150000);
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_EQ(connection->connected_since_time(), 0);
++pi3_client_statistics_count;
});
@@ -1022,6 +1034,9 @@
if (connection->state() != message_bridge::State::CONNECTED) {
return false;
}
+ EXPECT_TRUE(connection->has_boot_uuid());
+ EXPECT_TRUE(connection->has_connected_since_time());
+ EXPECT_TRUE(connection->has_connection_count());
}
return true;
}
@@ -1034,15 +1049,42 @@
if (connection->state() == message_bridge::State::CONNECTED) {
return false;
}
+ EXPECT_FALSE(connection->has_boot_uuid());
+ EXPECT_FALSE(connection->has_connected_since_time());
} else {
if (connection->state() != message_bridge::State::CONNECTED) {
return false;
}
+ EXPECT_TRUE(connection->has_boot_uuid());
+ EXPECT_TRUE(connection->has_connected_since_time());
+ EXPECT_TRUE(connection->has_connection_count());
}
}
return true;
}
+int ConnectedCount(const message_bridge::ClientStatistics *client_statistics,
+ std::string_view target) {
+ for (const message_bridge::ClientConnection *connection :
+ *client_statistics->connections()) {
+ if (connection->node()->name()->string_view() == target) {
+ return connection->connection_count();
+ }
+ }
+ return 0;
+}
+
+int ConnectedCount(const message_bridge::ServerStatistics *server_statistics,
+ std::string_view target) {
+ for (const message_bridge::ServerConnection *connection :
+ *server_statistics->connections()) {
+ if (connection->node()->name()->string_view() == target) {
+ return connection->connection_count();
+ }
+ }
+ return 0;
+}
+
// Test that disconnecting nodes actually disconnects them.
TEST_P(RemoteMessageSimulatedEventLoopTest, MultinodeDisconnect) {
SimulatedEventLoopFactory simulated_event_loop_factory(&config.message());
@@ -1247,6 +1289,33 @@
simulated_event_loop_factory.RunFor(chrono::seconds(2));
+ EXPECT_TRUE(pi1_server_statistics_fetcher.Fetch());
+ EXPECT_TRUE(pi1_client_statistics_fetcher.Fetch());
+ EXPECT_TRUE(pi2_server_statistics_fetcher.Fetch());
+ EXPECT_TRUE(pi2_client_statistics_fetcher.Fetch());
+ EXPECT_TRUE(pi3_server_statistics_fetcher.Fetch());
+ EXPECT_TRUE(pi3_client_statistics_fetcher.Fetch());
+
+ EXPECT_EQ(ConnectedCount(pi1_server_statistics_fetcher.get(), "pi3"), 2u)
+ << " : " << aos::FlatbufferToJson(pi1_server_statistics_fetcher.get());
+ EXPECT_EQ(ConnectedCount(pi1_server_statistics_fetcher.get(), "pi2"), 1u)
+ << " : " << aos::FlatbufferToJson(pi1_server_statistics_fetcher.get());
+ EXPECT_EQ(ConnectedCount(pi1_client_statistics_fetcher.get(), "pi3"), 1u)
+ << " : " << aos::FlatbufferToJson(pi1_client_statistics_fetcher.get());
+ EXPECT_EQ(ConnectedCount(pi1_client_statistics_fetcher.get(), "pi2"), 1u)
+ << " : " << aos::FlatbufferToJson(pi1_client_statistics_fetcher.get());
+
+ EXPECT_EQ(ConnectedCount(pi2_server_statistics_fetcher.get(), "pi1"), 1u)
+ << " : " << aos::FlatbufferToJson(pi2_server_statistics_fetcher.get());
+ EXPECT_EQ(ConnectedCount(pi2_client_statistics_fetcher.get(), "pi1"), 1u)
+ << " : " << aos::FlatbufferToJson(pi2_client_statistics_fetcher.get());
+
+ EXPECT_EQ(ConnectedCount(pi3_server_statistics_fetcher.get(), "pi1"), 1u)
+ << " : " << aos::FlatbufferToJson(pi3_server_statistics_fetcher.get());
+ EXPECT_EQ(ConnectedCount(pi3_client_statistics_fetcher.get(), "pi1"), 2u)
+ << " : " << aos::FlatbufferToJson(pi3_client_statistics_fetcher.get());
+
+
EXPECT_EQ(pi1_pong_counter.count(), 601u);
EXPECT_EQ(pi2_pong_counter.count(), 601u);
@@ -1270,22 +1339,16 @@
EXPECT_EQ(CountAll(remote_timestamps_pi2_on_pi1), 661);
EXPECT_EQ(CountAll(remote_timestamps_pi1_on_pi2), 661);
- EXPECT_TRUE(pi1_server_statistics_fetcher.Fetch());
EXPECT_TRUE(AllConnected(pi1_server_statistics_fetcher.get()))
<< " : " << aos::FlatbufferToJson(pi1_server_statistics_fetcher.get());
- EXPECT_TRUE(pi1_client_statistics_fetcher.Fetch());
EXPECT_TRUE(AllConnected(pi1_client_statistics_fetcher.get()))
<< " : " << aos::FlatbufferToJson(pi1_client_statistics_fetcher.get());
- EXPECT_TRUE(pi2_server_statistics_fetcher.Fetch());
EXPECT_TRUE(AllConnected(pi2_server_statistics_fetcher.get()))
<< " : " << aos::FlatbufferToJson(pi2_server_statistics_fetcher.get());
- EXPECT_TRUE(pi2_client_statistics_fetcher.Fetch());
EXPECT_TRUE(AllConnected(pi2_client_statistics_fetcher.get()))
<< " : " << aos::FlatbufferToJson(pi2_client_statistics_fetcher.get());
- EXPECT_TRUE(pi3_server_statistics_fetcher.Fetch());
EXPECT_TRUE(AllConnected(pi3_server_statistics_fetcher.get()))
<< " : " << aos::FlatbufferToJson(pi3_server_statistics_fetcher.get());
- EXPECT_TRUE(pi3_client_statistics_fetcher.Fetch());
EXPECT_TRUE(AllConnected(pi3_client_statistics_fetcher.get()))
<< " : " << aos::FlatbufferToJson(pi3_client_statistics_fetcher.get());
}
@@ -1563,10 +1626,13 @@
int pi1_server_statistics_count = 0;
bool first_pi1_server_statistics = true;
+ int boot_number = 0;
+ monotonic_clock::time_point expected_connection_time = pi1->monotonic_now();
pi1_remote_timestamp->MakeWatcher(
- "/pi1/aos", [&pi1_server_statistics_count, &expected_boot_uuid,
- &first_pi1_server_statistics](
- const message_bridge::ServerStatistics &stats) {
+ "/pi1/aos",
+ [&pi1_server_statistics_count, &expected_boot_uuid,
+ &expected_connection_time, &first_pi1_server_statistics,
+ &boot_number](const message_bridge::ServerStatistics &stats) {
VLOG(1) << "pi1 ServerStatistics " << FlatbufferToJson(&stats);
for (const message_bridge::ServerConnection *connection :
*stats.connections()) {
@@ -1582,6 +1648,10 @@
EXPECT_EQ(expected_boot_uuid,
UUID::FromString(connection->boot_uuid()))
<< " : Got " << aos::FlatbufferToJson(&stats);
+ EXPECT_EQ(monotonic_clock::time_point(chrono::nanoseconds(
+ connection->connected_since_time())),
+ expected_connection_time);
+ EXPECT_EQ(boot_number + 1, connection->connection_count());
++pi1_server_statistics_count;
}
}
@@ -1590,7 +1660,8 @@
int pi1_client_statistics_count = 0;
pi1_remote_timestamp->MakeWatcher(
- "/pi1/aos", [&pi1_client_statistics_count](
+ "/pi1/aos", [&pi1_client_statistics_count, &expected_boot_uuid,
+ &expected_connection_time, &boot_number](
const message_bridge::ClientStatistics &stats) {
VLOG(1) << "pi1 ClientStatistics " << FlatbufferToJson(&stats);
for (const message_bridge::ClientConnection *connection :
@@ -1598,18 +1669,33 @@
EXPECT_EQ(connection->state(), message_bridge::State::CONNECTED);
if (connection->node()->name()->string_view() == "pi2") {
++pi1_client_statistics_count;
+ EXPECT_EQ(expected_boot_uuid,
+ UUID::FromString(connection->boot_uuid()))
+ << " : Got " << aos::FlatbufferToJson(&stats);
+ EXPECT_EQ(monotonic_clock::time_point(chrono::nanoseconds(
+ connection->connected_since_time())),
+ expected_connection_time);
+ EXPECT_EQ(boot_number + 1, connection->connection_count());
+ } else {
+ EXPECT_EQ(connection->connected_since_time(), 0);
+ EXPECT_EQ(1, connection->connection_count());
}
}
});
// Confirm that reboot changes the UUID.
- pi2->OnShutdown([&expected_boot_uuid, pi2, pi2_boot1]() {
- expected_boot_uuid = pi2_boot1;
- LOG(INFO) << "OnShutdown triggered for pi2";
- pi2->OnStartup([&expected_boot_uuid, pi2]() {
- EXPECT_EQ(expected_boot_uuid, pi2->boot_uuid());
- });
- });
+ pi2->OnShutdown(
+ [&expected_boot_uuid, &boot_number, &expected_connection_time, pi1, pi2,
+ pi2_boot1]() {
+ expected_boot_uuid = pi2_boot1;
+ ++boot_number;
+ LOG(INFO) << "OnShutdown triggered for pi2";
+ pi2->OnStartup(
+ [&expected_boot_uuid, &expected_connection_time, pi1, pi2]() {
+ EXPECT_EQ(expected_boot_uuid, pi2->boot_uuid());
+ expected_connection_time = pi1->monotonic_now();
+ });
+ });
// Let a couple of ServerStatistics messages show up before rebooting.
factory.RunFor(chrono::milliseconds(2002));
diff --git a/aos/events/simulated_network_bridge.cc b/aos/events/simulated_network_bridge.cc
index 96a614d..4c9208e 100644
--- a/aos/events/simulated_network_bridge.cc
+++ b/aos/events/simulated_network_bridge.cc
@@ -257,9 +257,9 @@
fetcher_->context().queue_index, fetcher_->context().source_boot_uuid));
// And simulate message_bridge's offset recovery.
- client_status_->SampleFilter(client_index_,
- fetcher_->context().monotonic_event_time,
- sender_->monotonic_sent_time());
+ client_status_->SampleFilter(
+ client_index_, fetcher_->context().monotonic_event_time,
+ sender_->monotonic_sent_time(), fetcher_->context().source_boot_uuid);
client_connection_->mutate_received_packets(
client_connection_->received_packets() + 1);
@@ -631,9 +631,16 @@
for (ServerConnection *connection : server_status->server_connection()) {
if (connection) {
if (boot_uuids_[node_index] != UUID::Zero()) {
- connection->mutate_state(server_state_[node_index]);
+ switch (server_state_[node_index]) {
+ case message_bridge::State::DISCONNECTED:
+ server_status->Disconnect(node_index);
+ break;
+ case message_bridge::State::CONNECTED:
+ server_status->Connect(node_index, event_loop->monotonic_now());
+ break;
+ }
} else {
- connection->mutate_state(message_bridge::State::DISCONNECTED);
+ server_status->Disconnect(node_index);
}
}
++node_index;
@@ -666,9 +673,18 @@
const size_t client_node_index = configuration::GetNodeIndex(
node_factory_->configuration(), client_node);
if (boot_uuids_[client_node_index] != UUID::Zero()) {
- client_connection->mutate_state(client_state_[client_node_index]);
+ if (client_connection->state() != client_state_[client_node_index]) {
+ switch (client_state_[client_node_index]) {
+ case message_bridge::State::DISCONNECTED:
+ client_status->Disconnect(i);
+ break;
+ case message_bridge::State::CONNECTED:
+ client_status->Connect(i);
+ break;
+ }
+ }
} else {
- client_connection->mutate_state(message_bridge::State::DISCONNECTED);
+ client_status->Disconnect(i);
}
}
diff --git a/aos/events/simulated_network_bridge.h b/aos/events/simulated_network_bridge.h
index 231bba1..3a7a4e8 100644
--- a/aos/events/simulated_network_bridge.h
+++ b/aos/events/simulated_network_bridge.h
@@ -86,7 +86,6 @@
SetEventLoop(node_factory_->MakeEventLoop("message_bridge"));
}
- void ClearEventLoop() { SetEventLoop(nullptr); }
void SetEventLoop(std::unique_ptr<aos::EventLoop> loop);
void SetSendData(std::function<void(const Context &)> fn) {
@@ -109,19 +108,38 @@
ServerConnection *connection =
server_status->FindServerConnection(node);
if (connection) {
- connection->mutate_state(server_state_[node_index]);
- server_status->ResetFilter(node_index);
- server_status->SetBootUUID(node_index, boot_uuid);
+ if (boot_uuid == UUID::Zero()) {
+ server_status->Disconnect(node_index);
+ server_status->ResetFilter(node_index);
+ } else {
+ switch (server_state_[node_index]) {
+ case message_bridge::State::DISCONNECTED:
+ server_status->Disconnect(node_index);
+ break;
+ case message_bridge::State::CONNECTED:
+ server_status->Connect(node_index, event_loop->monotonic_now());
+ break;
+ }
+ server_status->ResetFilter(node_index);
+ server_status->SetBootUUID(node_index, boot_uuid);
+ }
}
}
if (client_status) {
const int client_index =
client_status->FindClientIndex(node->name()->string_view());
- ClientConnection *client_connection =
- client_status->GetClientConnection(client_index);
- if (client_connection) {
- client_status->SampleReset(client_index);
- client_connection->mutate_state(client_state_[node_index]);
+ client_status->SampleReset(client_index);
+ if (boot_uuid == UUID::Zero()) {
+ client_status->Disconnect(client_index);
+ } else {
+ switch (client_state_[node_index]) {
+ case message_bridge::State::CONNECTED:
+ client_status->Connect(client_index);
+ break;
+ case message_bridge::State::DISCONNECTED:
+ client_status->Disconnect(client_index);
+ break;
+ }
}
}
}
@@ -135,7 +153,17 @@
server_status->FindServerConnection(destination);
if (connection == nullptr) return;
- connection->mutate_state(state);
+ if (state == connection->state()) {
+ return;
+ }
+ switch (state) {
+ case message_bridge::State::DISCONNECTED:
+ server_status->Disconnect(node_index);
+ break;
+ case message_bridge::State::CONNECTED:
+ server_status->Connect(node_index, event_loop->monotonic_now());
+ break;
+ }
}
}
@@ -144,12 +172,23 @@
configuration::GetNodeIndex(node_factory_->configuration(), source);
client_state_[node_index] = state;
if (client_status) {
+ const int client_index =
+ client_status->FindClientIndex(source->name()->string_view());
ClientConnection *connection =
client_status->GetClientConnection(source);
- if (connection == nullptr) return;
-
- connection->mutate_state(state);
+ // TODO(austin): Are there cases where we want to dedup 2 CONNECTED
+ // calls?
+ if (connection->state() != state) {
+ switch (state) {
+ case message_bridge::State::CONNECTED:
+ client_status->Connect(client_index);
+ break;
+ case message_bridge::State::DISCONNECTED:
+ client_status->Disconnect(client_index);
+ break;
+ }
+ }
}
}
diff --git a/aos/network/message_bridge_client.fbs b/aos/network/message_bridge_client.fbs
index 6efe3a9..40c0698 100644
--- a/aos/network/message_bridge_client.fbs
+++ b/aos/network/message_bridge_client.fbs
@@ -25,7 +25,16 @@
// (indicates congestion)
partial_deliveries:uint (id: 5);
- // TODO(austin): Per channel counts?
+ // Boot UUID of the server.
+ boot_uuid:string (id: 6);
+
+ // Time at which we connected to the server as nanoseconds on the local
+ // monotonic clock. This is not populated when not connected, and defaults
+ // to monotonic_clock::min_time.
+ connected_since_time:int64 = -9223372036854775808 (id: 7);
+
+ // Number of times we've established a connection to the server.
+ connection_count:uint (id: 8);
}
// Statistics for all clients.
diff --git a/aos/network/message_bridge_client_lib.cc b/aos/network/message_bridge_client_lib.cc
index f535c4f..9001cce 100644
--- a/aos/network/message_bridge_client_lib.cc
+++ b/aos/network/message_bridge_client_lib.cc
@@ -208,16 +208,14 @@
// the priority scheduler. This only needs to be done once per stream.
client_.SetPriorityScheduler(assoc_id);
- connection_->mutate_state(State::CONNECTED);
- client_status_->SampleReset(client_index_);
+ client_status_->Connect(client_index_);
}
void SctpClientConnection::NodeDisconnected() {
connect_timer_->Setup(
event_loop_->monotonic_now() + chrono::milliseconds(100),
chrono::milliseconds(100));
- connection_->mutate_state(State::DISCONNECTED);
- connection_->mutate_monotonic_offset(0);
+ client_status_->Disconnect(client_index_);
client_status_->SampleReset(client_index_);
}
@@ -270,7 +268,8 @@
client_index_,
monotonic_clock::time_point(
chrono::nanoseconds(remote_data->monotonic_sent_time())),
- sender->monotonic_sent_time());
+ sender->monotonic_sent_time(),
+ UUID::FromVector(remote_data->boot_uuid()));
if (stream_reply_with_timestamp_[stream]) {
// TODO(austin): Send back less if we are only acking. Maybe only a
diff --git a/aos/network/message_bridge_client_status.cc b/aos/network/message_bridge_client_status.cc
index 357026a..aa2484d 100644
--- a/aos/network/message_bridge_client_status.cc
+++ b/aos/network/message_bridge_client_status.cc
@@ -34,6 +34,9 @@
connection_builder.add_duplicate_packets(0);
connection_builder.add_monotonic_offset(0);
connection_builder.add_partial_deliveries(0);
+ connection_builder.add_connected_since_time(
+ monotonic_clock::min_time.time_since_epoch().count());
+ connection_builder.add_connection_count(0);
connection_offsets.emplace_back(connection_builder.Finish());
}
flatbuffers::Offset<
@@ -58,6 +61,7 @@
client_connection_offsets_.reserve(
statistics_.message().connections()->size());
filters_.resize(statistics_.message().connections()->size());
+ uuids_.resize(statistics_.message().connections()->size(), UUID::Zero());
statistics_timer_ = event_loop_->AddTimer([this]() { SendStatistics(); });
statistics_timer_->set_name("statistics");
@@ -69,6 +73,26 @@
});
}
+void MessageBridgeClientStatus::Disconnect(int client_index) {
+ ClientConnection *connection = GetClientConnection(client_index);
+
+ connection->mutate_state(State::DISCONNECTED);
+ connection->mutate_connected_since_time(
+ monotonic_clock::min_time.time_since_epoch().count());
+ connection->mutate_monotonic_offset(0);
+
+ uuids_[client_index] = UUID::Zero();
+}
+
+void MessageBridgeClientStatus::Connect(int client_index) {
+ ClientConnection *connection = GetClientConnection(client_index);
+
+ connection->mutate_state(State::CONNECTED);
+ connection->mutate_connected_since_time(
+ event_loop_->monotonic_now().time_since_epoch().count());
+ connection->mutate_connection_count(connection->connection_count() + 1);
+}
+
void MessageBridgeClientStatus::SendStatistics() {
if (!send_) {
return;
@@ -78,14 +102,23 @@
aos::Sender<ClientStatistics>::Builder builder = sender_.MakeBuilder();
client_connection_offsets_.clear();
- for (const ClientConnection *connection :
- *statistics_.message().connections()) {
+ for (size_t client_index = 0;
+ client_index < statistics_.message().connections()->size();
+ ++client_index) {
+ const ClientConnection *connection =
+ statistics_.message().connections()->Get(client_index);
flatbuffers::Offset<flatbuffers::String> node_name_offset =
builder.fbb()->CreateString(connection->node()->name()->string_view());
Node::Builder node_builder = builder.MakeBuilder<Node>();
node_builder.add_name(node_name_offset);
flatbuffers::Offset<Node> node_offset = node_builder.Finish();
+ flatbuffers::Offset<flatbuffers::String> uuid_offset = 0;
+
+ if (uuids_[client_index] != UUID::Zero()) {
+ uuid_offset = uuids_[client_index].PackString(builder.fbb());
+ }
+
ClientConnection::Builder client_connection_builder =
builder.MakeBuilder<ClientConnection>();
@@ -97,9 +130,24 @@
client_connection_builder.add_duplicate_packets(
connection->duplicate_packets());
}
+
+ if (connection->connected_since_time() !=
+ monotonic_clock::min_time.time_since_epoch().count()) {
+ client_connection_builder.add_connected_since_time(
+ connection->connected_since_time());
+ }
+
+ if (connection->connection_count() != 0) {
+ client_connection_builder.add_connection_count(
+ connection->connection_count());
+ }
client_connection_builder.add_partial_deliveries(
connection->partial_deliveries());
+ if (!uuid_offset.IsNull()) {
+ client_connection_builder.add_boot_uuid(uuid_offset);
+ }
+
// Strip out the monotonic offset if it isn't populated.
TimestampFilter *filter = &filters_[client_connection_offsets_.size()];
if (filter->has_sample()) {
@@ -152,7 +200,8 @@
void MessageBridgeClientStatus::SampleFilter(
int client_index,
const aos::monotonic_clock::time_point monotonic_sent_time,
- const aos::monotonic_clock::time_point monotonic_delivered_time) {
+ const aos::monotonic_clock::time_point monotonic_delivered_time,
+ const UUID &uuid) {
TimestampFilter *filter = &filters_[client_index];
const std::chrono::nanoseconds offset =
@@ -164,6 +213,8 @@
filter->set_base_offset(offset);
}
+ uuids_[client_index] = uuid;
+
// We can now measure the latency!
filter->Sample(monotonic_delivered_time, offset);
}
diff --git a/aos/network/message_bridge_client_status.h b/aos/network/message_bridge_client_status.h
index d38244a..9c21169 100644
--- a/aos/network/message_bridge_client_status.h
+++ b/aos/network/message_bridge_client_status.h
@@ -42,11 +42,17 @@
void SampleFilter(
int client_index,
const aos::monotonic_clock::time_point monotonic_sent_time,
- const aos::monotonic_clock::time_point monotonic_delivered_time);
+ const aos::monotonic_clock::time_point monotonic_delivered_time,
+ const UUID &uuid);
// Clears out the filter state.
void SampleReset(int client_index) { filters_[client_index].Reset(); }
+ // Disconnects the client.
+ void Disconnect(int client_index);
+ // Connects the client.
+ void Connect(int client_index);
+
// Disables sending out any statistics messages.
void DisableStatistics();
// Enables sending out any statistics messages.
@@ -73,6 +79,8 @@
std::vector<TimestampFilter> filters_;
+ std::vector<UUID> uuids_;
+
// If true, send out the messages.
bool send_ = true;
};
diff --git a/aos/network/message_bridge_server.fbs b/aos/network/message_bridge_server.fbs
index 128f1f3..031f801 100644
--- a/aos/network/message_bridge_server.fbs
+++ b/aos/network/message_bridge_server.fbs
@@ -34,7 +34,13 @@
// (indicates congestion)
partial_deliveries:uint (id: 6);
- // TODO(austin): Per channel counts?
+ // Time at which we connected to the client as nanoseconds on the local
+ // monotonic clock. This is not populated when not connected, and defaults
+ // to monotonic_clock::min_time.
+ connected_since_time:int64 = -9223372036854775808 (id: 7);
+
+ // Number of times we've established a connection to the server.
+ connection_count:uint (id: 8);
}
// Statistics for all connections to all the clients.
diff --git a/aos/network/message_bridge_server_lib.cc b/aos/network/message_bridge_server_lib.cc
index dbca3fb..2ca8e48 100644
--- a/aos/network/message_bridge_server_lib.cc
+++ b/aos/network/message_bridge_server_lib.cc
@@ -171,10 +171,11 @@
void ChannelState::AddPeer(const Connection *connection, int node_index,
ServerConnection *server_connection_statistics,
+ MessageBridgeServerStatus *server_status,
bool logged_remotely,
aos::Sender<RemoteMessage> *timestamp_logger) {
peers_.emplace_back(connection, node_index, server_connection_statistics,
- logged_remotely, timestamp_logger);
+ server_status, logged_remotely, timestamp_logger);
}
int ChannelState::NodeDisconnected(sctp_assoc_t assoc_id) {
@@ -183,7 +184,6 @@
if (peer.sac_assoc_id == assoc_id) {
// TODO(austin): This will not handle multiple clients from
// a single node. But that should be rare.
- peer.server_connection_statistics->mutate_state(State::DISCONNECTED);
peer.sac_assoc_id = 0;
peer.stream = 0;
return peer.node_index;
@@ -192,8 +192,9 @@
return -1;
}
-int ChannelState::NodeConnected(const Node *node, sctp_assoc_t assoc_id,
- int stream, SctpServer *server) {
+int ChannelState::NodeConnected(
+ const Node *node, sctp_assoc_t assoc_id, int stream, SctpServer *server,
+ aos::monotonic_clock::time_point monotonic_now) {
VLOG(1) << "Connected to assoc_id: " << assoc_id << " for stream " << stream;
for (ChannelState::Peer &peer : peers_) {
if (peer.connection->name()->string_view() == node->name()->string_view()) {
@@ -210,7 +211,8 @@
peer.sac_assoc_id = assoc_id;
peer.stream = stream;
- peer.server_connection_statistics->mutate_state(State::CONNECTED);
+ peer.server_status->Connect(peer.node_index, monotonic_now);
+
server->SetStreamPriority(assoc_id, stream, peer.connection->priority());
if (last_message_fetcher_ && peer.connection->time_to_live() == 0) {
last_message_fetcher_->Fetch();
@@ -327,6 +329,7 @@
connection->name()->string_view()),
server_status_.FindServerConnection(
connection->name()->string_view()),
+ &server_status_,
configuration::ChannelMessageIsLoggedOnNode(channel, other_node),
delivery_time_is_logged
? timestamp_loggers_.SenderForChannel(channel, connection)
@@ -396,6 +399,7 @@
->Get(node_index)
->name()
->string_view();
+ server_status_.Disconnect(node_index);
server_status_.ResetFilter(node_index);
server_status_.ClearBootUUID(node_index);
server_status_.ResetPartialDeliveries(node_index);
@@ -456,6 +460,7 @@
CHECK_LE(connect->channels_to_transfer()->size(),
static_cast<size_t>(max_channels()))
<< ": Client has more channels than we do";
+ monotonic_clock::time_point monotonic_now = event_loop_->monotonic_now();
// Account for the control channel and delivery times channel.
size_t channel_index = kControlStreams();
@@ -469,7 +474,7 @@
if (channel_state->Matches(channel)) {
node_index = channel_state->NodeConnected(
connect->node(), message->header.rcvinfo.rcv_assoc_id,
- channel_index, &server_);
+ channel_index, &server_, monotonic_now);
CHECK_NE(node_index, -1);
matched = true;
diff --git a/aos/network/message_bridge_server_lib.h b/aos/network/message_bridge_server_lib.h
index 6f96c02..e70d34a 100644
--- a/aos/network/message_bridge_server_lib.h
+++ b/aos/network/message_bridge_server_lib.h
@@ -38,11 +38,12 @@
struct Peer {
Peer(const Connection *new_connection, int new_node_index,
ServerConnection *new_server_connection_statistics,
- bool new_logged_remotely,
+ MessageBridgeServerStatus *new_server_status, bool new_logged_remotely,
aos::Sender<RemoteMessage> *new_timestamp_logger)
: connection(new_connection),
node_index(new_node_index),
server_connection_statistics(new_server_connection_statistics),
+ server_status(new_server_status),
timestamp_logger(new_timestamp_logger),
logged_remotely(new_logged_remotely) {}
@@ -53,6 +54,7 @@
const aos::Connection *connection;
const int node_index;
ServerConnection *server_connection_statistics;
+ MessageBridgeServerStatus *server_status;
aos::Sender<RemoteMessage> *timestamp_logger = nullptr;
// If true, this message will be logged on a receiving node. We need to
@@ -64,12 +66,13 @@
// Returns the node index which [dis]connected, or -1 if it didn't match.
int NodeDisconnected(sctp_assoc_t assoc_id);
int NodeConnected(const Node *node, sctp_assoc_t assoc_id, int stream,
- SctpServer *server);
+ SctpServer *server,
+ aos::monotonic_clock::time_point monotonic_now);
// Adds a new peer.
void AddPeer(const Connection *connection, int node_index,
ServerConnection *server_connection_statistics,
- bool logged_remotely,
+ MessageBridgeServerStatus *server_status, bool logged_remotely,
aos::Sender<RemoteMessage> *timestamp_logger);
// Returns true if this channel has the same name and type as the other
diff --git a/aos/network/message_bridge_server_status.cc b/aos/network/message_bridge_server_status.cc
index 09355b2..c82158b 100644
--- a/aos/network/message_bridge_server_status.cc
+++ b/aos/network/message_bridge_server_status.cc
@@ -37,6 +37,9 @@
connection_builder.add_sent_packets(0);
connection_builder.add_monotonic_offset(0);
connection_builder.add_partial_deliveries(0);
+ connection_builder.add_connected_since_time(
+ monotonic_clock::min_time.time_since_epoch().count());
+ connection_builder.add_connection_count(0);
connection_offsets.emplace_back(connection_builder.Finish());
}
flatbuffers::Offset<
@@ -162,6 +165,26 @@
server_connection_[node_index]->mutate_monotonic_offset(0);
}
+void MessageBridgeServerStatus::Connect(
+ int node_index, monotonic_clock::time_point monotonic_now) {
+ server_connection_[node_index]->mutate_state(State::CONNECTED);
+ // Only count connections if the timestamp changes. This deduplicates
+ // multiple channel connections at the same point in time.
+ if (server_connection_[node_index]->connected_since_time() !=
+ monotonic_now.time_since_epoch().count()) {
+ server_connection_[node_index]->mutate_connection_count(
+ server_connection_[node_index]->connection_count() + 1);
+ server_connection_[node_index]->mutate_connected_since_time(
+ monotonic_now.time_since_epoch().count());
+ }
+}
+
+void MessageBridgeServerStatus::Disconnect(int node_index) {
+ server_connection_[node_index]->mutate_state(State::DISCONNECTED);
+ server_connection_[node_index]->mutate_connected_since_time(
+ aos::monotonic_clock::min_time.time_since_epoch().count());
+}
+
void MessageBridgeServerStatus::SendStatistics() {
if (!send_) return;
aos::Sender<ServerStatistics>::Builder builder = sender_.MakeBuilder();
@@ -197,6 +220,17 @@
server_connection_builder.add_partial_deliveries(
partial_deliveries_[node_index]);
+ if (connection->connected_since_time() !=
+ monotonic_clock::min_time.time_since_epoch().count()) {
+ server_connection_builder.add_connected_since_time(
+ connection->connected_since_time());
+ }
+
+ if (connection->connection_count() != 0) {
+ server_connection_builder.add_connection_count(
+ connection->connection_count());
+ }
+
// TODO(austin): If it gets stale, drop it too.
if (!filters_[node_index].MissingSamples()) {
server_connection_builder.add_monotonic_offset(
diff --git a/aos/network/message_bridge_server_status.h b/aos/network/message_bridge_server_status.h
index 3a1c9ed..feb1b05 100644
--- a/aos/network/message_bridge_server_status.h
+++ b/aos/network/message_bridge_server_status.h
@@ -45,6 +45,9 @@
// Clears the boot UUID for the provided node.
void ClearBootUUID(int node_index);
+ void Connect(int node_index, monotonic_clock::time_point monotonic_now);
+ void Disconnect(int node_index);
+
// Returns the boot UUID for a node, or an empty string_view if there isn't
// one.
const UUID &BootUUID(int node_index) const { return boot_uuids_[node_index]; }
diff --git a/aos/network/message_bridge_test.cc b/aos/network/message_bridge_test.cc
index 8246c22..d2a9b05 100644
--- a/aos/network/message_bridge_test.cc
+++ b/aos/network/message_bridge_test.cc
@@ -439,7 +439,14 @@
pi2_client_event_loop->node()->name()->string_view()) {
if (connection->state() == State::CONNECTED) {
EXPECT_TRUE(connection->has_boot_uuid());
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_LT(monotonic_clock::time_point(chrono::nanoseconds(
+ connection->connected_since_time())),
+ monotonic_clock::now());
connected = true;
+ } else {
+ EXPECT_FALSE(connection->has_connection_count());
+ EXPECT_FALSE(connection->has_connected_since_time());
}
}
}
@@ -473,48 +480,95 @@
chrono::milliseconds(-1));
EXPECT_TRUE(connection->has_boot_uuid());
}
+
+ if (connection->state() == State::CONNECTED) {
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_LT(monotonic_clock::time_point(
+ chrono::nanoseconds(connection->connected_since_time())),
+ monotonic_clock::now());
+ } else {
+ EXPECT_FALSE(connection->has_connection_count());
+ EXPECT_FALSE(connection->has_connected_since_time());
+ }
}
});
int pi1_client_statistics_count = 0;
- ping_event_loop.MakeWatcher("/pi1/aos", [&pi1_client_statistics_count](
- const ClientStatistics &stats) {
- VLOG(1) << "/pi1/aos ClientStatistics " << FlatbufferToJson(&stats);
+ int pi1_connected_client_statistics_count = 0;
+ ping_event_loop.MakeWatcher(
+ "/pi1/aos",
+ [&pi1_client_statistics_count,
+ &pi1_connected_client_statistics_count](const ClientStatistics &stats) {
+ VLOG(1) << "/pi1/aos ClientStatistics " << FlatbufferToJson(&stats);
- for (const ClientConnection *connection : *stats.connections()) {
- if (connection->has_monotonic_offset()) {
- ++pi1_client_statistics_count;
- // It takes at least 10 microseconds to send a message between the
- // client and server. The min (filtered) time shouldn't be over 10
- // milliseconds on localhost. This might have to bump up if this is
- // proving flaky.
- EXPECT_LT(chrono::nanoseconds(connection->monotonic_offset()),
- chrono::milliseconds(10))
- << " " << connection->monotonic_offset()
- << "ns vs 10000ns on iteration " << pi1_client_statistics_count;
- EXPECT_GT(chrono::nanoseconds(connection->monotonic_offset()),
- chrono::microseconds(10))
- << " " << connection->monotonic_offset()
- << "ns vs 10000ns on iteration " << pi1_client_statistics_count;
- }
- }
- });
+ for (const ClientConnection *connection : *stats.connections()) {
+ if (connection->has_monotonic_offset()) {
+ ++pi1_client_statistics_count;
+ // It takes at least 10 microseconds to send a message between the
+ // client and server. The min (filtered) time shouldn't be over 10
+ // milliseconds on localhost. This might have to bump up if this is
+ // proving flaky.
+ EXPECT_LT(chrono::nanoseconds(connection->monotonic_offset()),
+ chrono::milliseconds(10))
+ << " " << connection->monotonic_offset()
+ << "ns vs 10000ns on iteration " << pi1_client_statistics_count;
+ EXPECT_GT(chrono::nanoseconds(connection->monotonic_offset()),
+ chrono::microseconds(10))
+ << " " << connection->monotonic_offset()
+ << "ns vs 10000ns on iteration " << pi1_client_statistics_count;
+ }
+ if (connection->state() == State::CONNECTED) {
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_LT(monotonic_clock::time_point(chrono::nanoseconds(
+ connection->connected_since_time())),
+ monotonic_clock::now());
+ // The first Connected message may not have a UUID in it since no
+ // data has flown. That's fine.
+ if (pi1_connected_client_statistics_count > 0) {
+ EXPECT_TRUE(connection->has_boot_uuid())
+ << ": " << aos::FlatbufferToJson(connection);
+ }
+ ++pi1_connected_client_statistics_count;
+ } else {
+ EXPECT_FALSE(connection->has_connection_count());
+ EXPECT_FALSE(connection->has_connected_since_time());
+ }
+ }
+ });
int pi2_client_statistics_count = 0;
- pong_event_loop.MakeWatcher("/pi2/aos", [&pi2_client_statistics_count](
- const ClientStatistics &stats) {
- VLOG(1) << "/pi2/aos ClientStatistics " << FlatbufferToJson(&stats);
+ int pi2_connected_client_statistics_count = 0;
+ pong_event_loop.MakeWatcher(
+ "/pi2/aos",
+ [&pi2_client_statistics_count,
+ &pi2_connected_client_statistics_count](const ClientStatistics &stats) {
+ VLOG(1) << "/pi2/aos ClientStatistics " << FlatbufferToJson(&stats);
- for (const ClientConnection *connection : *stats.connections()) {
- if (connection->has_monotonic_offset()) {
- ++pi2_client_statistics_count;
- EXPECT_LT(chrono::nanoseconds(connection->monotonic_offset()),
- chrono::milliseconds(10));
- EXPECT_GT(chrono::nanoseconds(connection->monotonic_offset()),
- chrono::microseconds(10));
- }
- }
- });
+ for (const ClientConnection *connection : *stats.connections()) {
+ if (connection->has_monotonic_offset()) {
+ ++pi2_client_statistics_count;
+ EXPECT_LT(chrono::nanoseconds(connection->monotonic_offset()),
+ chrono::milliseconds(10))
+ << ": got " << aos::FlatbufferToJson(connection);
+ EXPECT_GT(chrono::nanoseconds(connection->monotonic_offset()),
+ chrono::microseconds(10))
+ << ": got " << aos::FlatbufferToJson(connection);
+ }
+ if (connection->state() == State::CONNECTED) {
+ EXPECT_EQ(connection->connection_count(), 1u);
+ EXPECT_LT(monotonic_clock::time_point(chrono::nanoseconds(
+ connection->connected_since_time())),
+ monotonic_clock::now());
+ if (pi2_connected_client_statistics_count > 0) {
+ EXPECT_TRUE(connection->has_boot_uuid());
+ }
+ ++pi2_connected_client_statistics_count;
+ } else {
+ EXPECT_FALSE(connection->has_connection_count());
+ EXPECT_FALSE(connection->has_connected_since_time());
+ }
+ }
+ });
ping_event_loop.MakeWatcher("/pi1/aos", [](const Timestamp ×tamp) {
EXPECT_TRUE(timestamp.has_offsets());
@@ -756,6 +810,8 @@
pi2_server_statistics_fetcher->connections()->Get(0);
EXPECT_EQ(pi1_connection->state(), State::CONNECTED);
+ EXPECT_EQ(pi1_connection->connection_count(), 1u);
+ EXPECT_TRUE(pi1_connection->has_connected_since_time());
EXPECT_TRUE(pi1_connection->has_monotonic_offset());
EXPECT_LT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
chrono::milliseconds(1));
@@ -764,6 +820,8 @@
EXPECT_TRUE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
+ EXPECT_EQ(pi2_connection->connection_count(), 1u);
+ EXPECT_TRUE(pi2_connection->has_connected_since_time());
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
EXPECT_LT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
chrono::milliseconds(1));
@@ -787,11 +845,15 @@
pi2_server_statistics_fetcher->connections()->Get(0);
EXPECT_EQ(pi1_connection->state(), State::DISCONNECTED);
+ EXPECT_EQ(pi1_connection->connection_count(), 1u);
+ EXPECT_FALSE(pi1_connection->has_connected_since_time());
EXPECT_FALSE(pi1_connection->has_monotonic_offset());
EXPECT_FALSE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_FALSE(pi2_connection->has_monotonic_offset());
EXPECT_TRUE(pi2_connection->has_boot_uuid());
+ EXPECT_EQ(pi2_connection->connection_count(), 1u);
+ EXPECT_TRUE(pi2_connection->has_connected_since_time());
}
{
@@ -809,19 +871,27 @@
pi2_server_statistics_fetcher->connections()->Get(0);
EXPECT_EQ(pi1_connection->state(), State::CONNECTED);
+ EXPECT_EQ(pi1_connection->connection_count(), 2u);
+ EXPECT_TRUE(pi1_connection->has_connected_since_time());
EXPECT_TRUE(pi1_connection->has_monotonic_offset());
EXPECT_LT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
- chrono::milliseconds(1));
+ chrono::milliseconds(1))
+ << ": " << FlatbufferToJson(pi1_connection);
EXPECT_GT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
- chrono::milliseconds(-1));
+ chrono::milliseconds(-1))
+ << ": " << FlatbufferToJson(pi1_connection);
EXPECT_TRUE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
+ EXPECT_EQ(pi2_connection->connection_count(), 1u);
+ EXPECT_TRUE(pi2_connection->has_connected_since_time());
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
EXPECT_LT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
- chrono::milliseconds(1));
+ chrono::milliseconds(1))
+ << ": " << FlatbufferToJson(pi2_connection);
EXPECT_GT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
- chrono::milliseconds(-1));
+ chrono::milliseconds(-1))
+ << ": " << FlatbufferToJson(pi2_connection);
EXPECT_TRUE(pi2_connection->has_boot_uuid());
StopPi2Client();
@@ -905,6 +975,8 @@
EXPECT_GT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
chrono::milliseconds(-1));
EXPECT_TRUE(pi1_connection->has_boot_uuid());
+ EXPECT_TRUE(pi1_connection->has_connected_since_time());
+ EXPECT_EQ(pi1_connection->connection_count(), 1u);
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
@@ -913,6 +985,8 @@
EXPECT_GT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
chrono::milliseconds(-1));
EXPECT_TRUE(pi2_connection->has_boot_uuid());
+ EXPECT_TRUE(pi2_connection->has_connected_since_time());
+ EXPECT_EQ(pi2_connection->connection_count(), 1u);
StopPi2Server();
}
@@ -931,9 +1005,15 @@
EXPECT_EQ(pi1_server_connection->state(), State::CONNECTED);
EXPECT_FALSE(pi1_server_connection->has_monotonic_offset());
+ EXPECT_TRUE(pi1_server_connection->has_connected_since_time());
+ EXPECT_EQ(pi1_server_connection->connection_count(), 1u);
+
EXPECT_TRUE(pi1_server_connection->has_boot_uuid());
EXPECT_EQ(pi1_client_connection->state(), State::DISCONNECTED);
EXPECT_FALSE(pi1_client_connection->has_monotonic_offset());
+ EXPECT_FALSE(pi1_client_connection->has_connected_since_time());
+ EXPECT_EQ(pi1_client_connection->connection_count(), 1u);
+ EXPECT_FALSE(pi1_client_connection->has_boot_uuid());
}
{
@@ -944,11 +1024,14 @@
// And confirm we are synchronized again.
EXPECT_TRUE(pi1_server_statistics_fetcher.Fetch());
EXPECT_TRUE(pi2_server_statistics_fetcher.Fetch());
+ EXPECT_TRUE(pi1_client_statistics_fetcher.Fetch());
const ServerConnection *const pi1_connection =
pi1_server_statistics_fetcher->connections()->Get(0);
const ServerConnection *const pi2_connection =
pi2_server_statistics_fetcher->connections()->Get(0);
+ const ClientConnection *const pi1_client_connection =
+ pi1_client_statistics_fetcher->connections()->Get(0);
EXPECT_EQ(pi1_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi1_connection->has_monotonic_offset());
@@ -958,6 +1041,11 @@
chrono::milliseconds(-1));
EXPECT_TRUE(pi1_connection->has_boot_uuid());
+ EXPECT_EQ(pi1_client_connection->state(), State::CONNECTED);
+ EXPECT_TRUE(pi1_client_connection->has_connected_since_time());
+ EXPECT_EQ(pi1_client_connection->connection_count(), 2u);
+ EXPECT_TRUE(pi1_client_connection->has_boot_uuid());
+
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
EXPECT_LT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
diff --git a/y2020/y2020_logger.json b/y2020/y2020_logger.json
index 0d97a7d..eba9996 100644
--- a/y2020/y2020_logger.json
+++ b/y2020/y2020_logger.json
@@ -141,6 +141,7 @@
"type": "aos.message_bridge.ClientStatistics",
"source_node": "logger",
"frequency": 10,
+ "max_size": 2000,
"num_senders": 2
},
{
diff --git a/y2020/y2020_roborio.json b/y2020/y2020_roborio.json
index 8bab794..a7f17b0 100644
--- a/y2020/y2020_roborio.json
+++ b/y2020/y2020_roborio.json
@@ -55,7 +55,7 @@
"type": "aos.message_bridge.ClientStatistics",
"source_node": "roborio",
"frequency": 15,
- "max_size": 736,
+ "max_size": 2000,
"num_senders": 2
},
{