Put remote boot UUID in ServerStatistics and RemoteMessage
This makes it so we can see when a remote reboots easily, and gives us a
stepping stone to putting it in the log file header.
RemoteMessage then goes to the logger, so we are ready to teach the
logger how to detect and handle reboots.
Change-Id: I943de46094b60535c92a677b430d6828933b820d
diff --git a/aos/network/connect.fbs b/aos/network/connect.fbs
index b593472..7ff2fbe 100644
--- a/aos/network/connect.fbs
+++ b/aos/network/connect.fbs
@@ -10,4 +10,9 @@
// The channels that we want transfered to this client.
channels_to_transfer:[Channel] (id: 1);
+
+ // The UUID that this node booted with.
+ boot_uuid: string (id: 2);
}
+
+root_type Connect;
diff --git a/aos/network/message_bridge_client_lib.cc b/aos/network/message_bridge_client_lib.cc
index 99b2ac1..38c3831 100644
--- a/aos/network/message_bridge_client_lib.cc
+++ b/aos/network/message_bridge_client_lib.cc
@@ -11,6 +11,7 @@
#include "aos/network/sctp_client.h"
#include "aos/network/timestamp_generated.h"
#include "aos/unique_malloc_ptr.h"
+#include "aos/util/file.h"
#include "gflags/gflags.h"
#include "glog/logging.h"
@@ -97,8 +98,9 @@
std::vector<SctpClientChannelState> *channels, int client_index,
MessageBridgeClientStatus *client_status)
: event_loop_(event_loop),
- connect_message_(MakeConnectMessage(event_loop->configuration(), my_node,
- remote_name)),
+ connect_message_(
+ MakeConnectMessage(event_loop->configuration(), my_node, remote_name,
+ event_loop->boot_uuid().string_view())),
message_reception_reply_(MakeMessageHeaderReply()),
remote_node_(CHECK_NOTNULL(
configuration::GetNode(event_loop->configuration(), remote_name))),
diff --git a/aos/network/message_bridge_protocol.cc b/aos/network/message_bridge_protocol.cc
index 8936da8..53102c2 100644
--- a/aos/network/message_bridge_protocol.cc
+++ b/aos/network/message_bridge_protocol.cc
@@ -13,12 +13,15 @@
aos::FlatbufferDetachedBuffer<aos::message_bridge::Connect> MakeConnectMessage(
const Configuration *config, const Node *my_node,
- std::string_view remote_name) {
+ std::string_view remote_name, std::string_view boot_uuid) {
CHECK(config->has_nodes()) << ": Config must have nodes to transfer.";
flatbuffers::FlatBufferBuilder fbb;
fbb.ForceDefaults(true);
+ flatbuffers::Offset<flatbuffers::String> boot_uuid_offset =
+ fbb.CreateString(boot_uuid);
+
flatbuffers::Offset<Node> node_offset =
RecursiveCopyFlatBuffer<Node>(my_node, &fbb);
const std::string_view node_name = my_node->name()->string_view();
@@ -46,6 +49,7 @@
Connect::Builder connect_builder(fbb);
connect_builder.add_channels_to_transfer(channels_offset);
connect_builder.add_node(node_offset);
+ connect_builder.add_boot_uuid(boot_uuid_offset);
fbb.Finish(connect_builder.Finish());
return fbb.Release();
diff --git a/aos/network/message_bridge_protocol.h b/aos/network/message_bridge_protocol.h
index 7b6a248..5759a29 100644
--- a/aos/network/message_bridge_protocol.h
+++ b/aos/network/message_bridge_protocol.h
@@ -33,7 +33,7 @@
// Builds up a subscription request for my_node to remote_name.
aos::FlatbufferDetachedBuffer<aos::message_bridge::Connect> MakeConnectMessage(
const Configuration *config, const Node *my_node,
- std::string_view remote_name);
+ std::string_view remote_name, std::string_view boot_uuid);
} // namespace message_bridge
} // namespace aos
diff --git a/aos/network/message_bridge_server.fbs b/aos/network/message_bridge_server.fbs
index 1be0796..fa73ba5 100644
--- a/aos/network/message_bridge_server.fbs
+++ b/aos/network/message_bridge_server.fbs
@@ -27,6 +27,9 @@
// monotonic time.
monotonic_offset:int64 (id: 4);
+ // Boot UUID of the client.
+ boot_uuid:string (id: 5);
+
// TODO(austin): Per channel counts?
}
diff --git a/aos/network/message_bridge_server_lib.cc b/aos/network/message_bridge_server_lib.cc
index d8a632e..1ac6cb5 100644
--- a/aos/network/message_bridge_server_lib.cc
+++ b/aos/network/message_bridge_server_lib.cc
@@ -94,8 +94,9 @@
// and flushes. Whee.
}
-void ChannelState::HandleDelivery(sctp_assoc_t rcv_assoc_id, uint16_t /*ssn*/,
- absl::Span<const uint8_t> data) {
+void ChannelState::HandleDelivery(
+ sctp_assoc_t rcv_assoc_id, uint16_t /*ssn*/, absl::Span<const uint8_t> data,
+ const MessageBridgeServerStatus &server_status) {
const logger::MessageHeader *message_header =
flatbuffers::GetRoot<logger::MessageHeader>(data.data());
for (Peer &peer : peers_) {
@@ -110,6 +111,10 @@
aos::Sender<RemoteMessage>::Builder builder =
peer.timestamp_logger->MakeBuilder();
+ flatbuffers::Offset<flatbuffers::String> boot_uuid_offset =
+ builder.fbb()->CreateString(
+ server_status.BootUUID(peer.node_index));
+
RemoteMessage::Builder message_header_builder =
builder.MakeBuilder<RemoteMessage>();
@@ -131,6 +136,7 @@
message_header->realtime_sent_time());
message_header_builder.add_remote_queue_index(
message_header->queue_index());
+ message_header_builder.add_boot_uuid(boot_uuid_offset);
builder.Send(message_header_builder.Finish());
}
@@ -271,7 +277,8 @@
MakeConnectMessage(event_loop->configuration(),
configuration::GetNode(event_loop->configuration(),
destination_node_name),
- event_loop->node()->name()->string_view())
+ event_loop->node()->name()->string_view(),
+ UUID::Zero().string_view())
.span()
.size());
VLOG(1) << "Connection to " << destination_node_name << " has size "
@@ -410,6 +417,7 @@
->name()
->string_view();
server_status_.ResetFilter(node_index);
+ server_status_.SetBootUUID(node_index, "");
}
}
@@ -484,6 +492,7 @@
}
}
server_status_.ResetFilter(node_index);
+ server_status_.SetBootUUID(node_index, connect->boot_uuid()->string_view());
VLOG(1) << "Resetting filters for " << node_index << " "
<< event_loop_->configuration()
->nodes()
@@ -500,7 +509,8 @@
->HandleDelivery(
message->header.rcvinfo.rcv_assoc_id,
message->header.rcvinfo.rcv_ssn,
- absl::Span<const uint8_t>(message->data(), message->size));
+ absl::Span<const uint8_t>(message->data(), message->size),
+ server_status_);
}
if (VLOG_IS_ON(1)) {
diff --git a/aos/network/message_bridge_server_lib.h b/aos/network/message_bridge_server_lib.h
index 04b0e16..55f3fea 100644
--- a/aos/network/message_bridge_server_lib.h
+++ b/aos/network/message_bridge_server_lib.h
@@ -82,7 +82,8 @@
// Handles reception of delivery times.
void HandleDelivery(sctp_assoc_t rcv_assoc_id, uint16_t ssn,
- absl::Span<const uint8_t> data);
+ absl::Span<const uint8_t> data,
+ const MessageBridgeServerStatus &server_status);
// Handles (by consuming) failure to deliver a message.
void HandleFailure(
diff --git a/aos/network/message_bridge_server_status.cc b/aos/network/message_bridge_server_status.cc
index 1f2de0b..c5c30d7 100644
--- a/aos/network/message_bridge_server_status.cc
+++ b/aos/network/message_bridge_server_status.cc
@@ -85,6 +85,11 @@
statistics_.message().connections()->size());
filters_.resize(event_loop->configuration()->nodes()->size());
+ boot_uuids_.resize(event_loop->configuration()->nodes()->size());
+ for (std::string &boot_uuid : boot_uuids_) {
+ // Make sure the memory gets allocated.
+ boot_uuid.reserve(UUID::kSize);
+ }
timestamp_fetchers_.resize(event_loop->configuration()->nodes()->size());
server_connection_.resize(event_loop->configuration()->nodes()->size());
@@ -129,6 +134,13 @@
node_name);
}
+void MessageBridgeServerStatus::SetBootUUID(int node_index,
+ std::string_view boot_uuid) {
+ boot_uuids_[node_index] = boot_uuid;
+ SendStatistics();
+ last_statistics_send_time_ = event_loop_->monotonic_now();
+}
+
void MessageBridgeServerStatus::ResetFilter(int node_index) {
filters_[node_index].Reset();
server_connection_[node_index]->mutate_monotonic_offset(0);
@@ -153,6 +165,12 @@
node_builder.add_name(node_name_offset);
flatbuffers::Offset<Node> node_offset = node_builder.Finish();
+ flatbuffers::Offset<flatbuffers::String> boot_uuid_offset;
+ if (!boot_uuids_[node_index].empty() &&
+ connection->state() == State::CONNECTED) {
+ boot_uuid_offset = builder.fbb()->CreateString(boot_uuids_[node_index]);
+ }
+
ServerConnection::Builder server_connection_builder =
builder.MakeBuilder<ServerConnection>();
server_connection_builder.add_node(node_offset);
@@ -167,6 +185,10 @@
connection->monotonic_offset());
}
+ if (!boot_uuid_offset.IsNull()) {
+ server_connection_builder.add_boot_uuid(boot_uuid_offset);
+ }
+
server_connection_offsets_.emplace_back(server_connection_builder.Finish());
}
diff --git a/aos/network/message_bridge_server_status.h b/aos/network/message_bridge_server_status.h
index 814fccb..0e70e24 100644
--- a/aos/network/message_bridge_server_status.h
+++ b/aos/network/message_bridge_server_status.h
@@ -40,6 +40,14 @@
// Resets the filter and clears the entry from the server statistics.
void ResetFilter(int node_index);
+ // Sets the boot UUID for the provided node.
+ void SetBootUUID(int node_index, std::string_view boot_uuid);
+
+ // Returns the boot UUID for a node, or an empty string_view if there isn't
+ // one.
+ std::string_view BootUUID(int node_index) const {
+ return boot_uuids_[node_index];
+ }
// Returns the ServerConnection message which is updated by the server.
ServerConnection *FindServerConnection(std::string_view node_name);
@@ -83,6 +91,9 @@
// Bidirectional filters for each connection.
std::vector<ClippedAverageFilter> filters_;
+ // List of UUIDs for each node.
+ std::vector<std::string> boot_uuids_;
+
// Sender for the timestamps that we are forwarding over the network.
aos::Sender<Timestamp> timestamp_sender_;
diff --git a/aos/network/message_bridge_test.cc b/aos/network/message_bridge_test.cc
index ed192b7..8588423 100644
--- a/aos/network/message_bridge_test.cc
+++ b/aos/network/message_bridge_test.cc
@@ -152,7 +152,7 @@
"/pi1/aos",
[&ping_count, &pi2_client_event_loop, &ping_sender,
&pi1_server_statistics_count](const ServerStatistics &stats) {
- LOG(INFO) << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi1/aos ServerStatistics " << FlatbufferToJson(&stats);
ASSERT_TRUE(stats.has_connections());
EXPECT_EQ(stats.connections()->size(), 1);
@@ -172,6 +172,7 @@
if (connection->node()->name()->string_view() ==
pi2_client_event_loop.node()->name()->string_view()) {
if (connection->state() == State::CONNECTED) {
+ EXPECT_TRUE(connection->has_boot_uuid());
connected = true;
}
}
@@ -193,7 +194,7 @@
int pi2_server_statistics_count = 0;
pong_event_loop.MakeWatcher("/pi2/aos", [&pi2_server_statistics_count](
const ServerStatistics &stats) {
- LOG(INFO) << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi2/aos ServerStatistics " << FlatbufferToJson(&stats);
for (const ServerConnection *connection : *stats.connections()) {
if (connection->has_monotonic_offset()) {
++pi2_server_statistics_count;
@@ -203,6 +204,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(connection->has_boot_uuid());
}
}
});
@@ -210,7 +212,7 @@
int pi1_client_statistics_count = 0;
ping_event_loop.MakeWatcher("/pi1/aos", [&pi1_client_statistics_count](
const ClientStatistics &stats) {
- LOG(INFO) << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi1/aos ClientStatistics " << FlatbufferToJson(&stats);
for (const ClientConnection *connection : *stats.connections()) {
if (connection->has_monotonic_offset()) {
@@ -230,7 +232,7 @@
int pi2_client_statistics_count = 0;
pong_event_loop.MakeWatcher("/pi2/aos", [&pi2_client_statistics_count](
const ClientStatistics &stats) {
- LOG(INFO) << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi2/aos ClientStatistics " << FlatbufferToJson(&stats);
for (const ClientConnection *connection : *stats.connections()) {
if (connection->has_monotonic_offset()) {
@@ -286,6 +288,8 @@
VLOG(1) << "/pi1/aos/remote_timestamps/pi2 RemoteMessage "
<< aos::FlatbufferToJson(&header);
+ EXPECT_TRUE(header.has_boot_uuid());
+
const aos::monotonic_clock::time_point header_monotonic_sent_time(
chrono::nanoseconds(header.monotonic_sent_time()));
const aos::realtime_clock::time_point header_realtime_sent_time(
@@ -462,29 +466,29 @@
// Wait until we are connected, then send.
pi1_test_event_loop.MakeWatcher(
"/pi1/aos", [](const ServerStatistics &stats) {
- LOG(INFO) << "pi1 ServerStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi1/aos ServerStatistics " << FlatbufferToJson(&stats);
});
pi2_test_event_loop.MakeWatcher(
"/pi2/aos", [](const ServerStatistics &stats) {
- LOG(INFO) << "pi2 ServerStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi2/aos ServerStatistics " << FlatbufferToJson(&stats);
});
pi1_test_event_loop.MakeWatcher(
"/pi1/aos", [](const ClientStatistics &stats) {
- LOG(INFO) << "pi1 ClientStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi1/aos ClientStatistics " << FlatbufferToJson(&stats);
});
pi2_test_event_loop.MakeWatcher(
"/pi2/aos", [](const ClientStatistics &stats) {
- LOG(INFO) << "pi2 ClientStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi2/aos ClientStatistics " << FlatbufferToJson(&stats);
});
pi1_test_event_loop.MakeWatcher("/pi1/aos", [](const Timestamp ×tamp) {
- LOG(INFO) << "pi1 Timestamp " << FlatbufferToJson(×tamp);
+ LOG(INFO) << "/pi1/aos Timestamp " << FlatbufferToJson(×tamp);
});
pi2_test_event_loop.MakeWatcher("/pi2/aos", [](const Timestamp ×tamp) {
- LOG(INFO) << "pi2 Timestamp " << FlatbufferToJson(×tamp);
+ LOG(INFO) << "/pi2/aos Timestamp " << FlatbufferToJson(×tamp);
});
// Start everything up. Pong is the only thing we don't know how to wait on,
@@ -533,6 +537,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
@@ -540,6 +545,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi2_connection->has_boot_uuid());
}
std::this_thread::sleep_for(std::chrono::seconds(2));
@@ -555,8 +561,10 @@
EXPECT_EQ(pi1_connection->state(), State::DISCONNECTED);
EXPECT_FALSE(pi1_connection->has_monotonic_offset());
+ EXPECT_FALSE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_FALSE(pi2_connection->has_monotonic_offset());
+ EXPECT_TRUE(pi2_connection->has_boot_uuid());
}
{
@@ -591,6 +599,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
@@ -598,6 +607,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi2_connection->has_boot_uuid());
}
// Shut everyone else down
@@ -665,31 +675,31 @@
// Wait until we are connected, then send.
pi1_test_event_loop.MakeWatcher(
"/pi1/aos", [](const ServerStatistics &stats) {
- LOG(INFO) << "pi1 ServerStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi1/aos ServerStatistics " << FlatbufferToJson(&stats);
});
// Confirm both client and server statistics messages have decent offsets in
// them.
pi2_test_event_loop.MakeWatcher(
"/pi2/aos", [](const ServerStatistics &stats) {
- LOG(INFO) << "pi2 ServerStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi2/aos ServerStatistics " << FlatbufferToJson(&stats);
});
pi1_test_event_loop.MakeWatcher(
"/pi1/aos", [](const ClientStatistics &stats) {
- LOG(INFO) << "pi1 ClientStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi1/aos ClientStatistics " << FlatbufferToJson(&stats);
});
pi2_test_event_loop.MakeWatcher(
"/pi2/aos", [](const ClientStatistics &stats) {
- LOG(INFO) << "pi2 ClientStatistics " << FlatbufferToJson(&stats);
+ LOG(INFO) << "/pi2/aos ClientStatistics " << FlatbufferToJson(&stats);
});
pi1_test_event_loop.MakeWatcher("/pi1/aos", [](const Timestamp ×tamp) {
- LOG(INFO) << "pi1 Timestamp " << FlatbufferToJson(×tamp);
+ LOG(INFO) << "/pi1/aos Timestamp " << FlatbufferToJson(×tamp);
});
pi2_test_event_loop.MakeWatcher("/pi2/aos", [](const Timestamp ×tamp) {
- LOG(INFO) << "pi2 Timestamp " << FlatbufferToJson(×tamp);
+ LOG(INFO) << "/pi2/aos Timestamp " << FlatbufferToJson(×tamp);
});
// Start everything up. Pong is the only thing we don't know how to wait on,
@@ -738,6 +748,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
@@ -745,6 +756,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi2_connection->has_boot_uuid());
}
std::this_thread::sleep_for(std::chrono::seconds(2));
@@ -761,6 +773,7 @@
EXPECT_EQ(pi1_server_connection->state(), State::CONNECTED);
EXPECT_FALSE(pi1_server_connection->has_monotonic_offset());
+ EXPECT_TRUE(pi1_server_connection->has_boot_uuid());
EXPECT_EQ(pi1_client_connection->state(), State::DISCONNECTED);
EXPECT_FALSE(pi1_client_connection->has_monotonic_offset());
}
@@ -797,6 +810,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi1_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi1_connection->has_boot_uuid());
EXPECT_EQ(pi2_connection->state(), State::CONNECTED);
EXPECT_TRUE(pi2_connection->has_monotonic_offset());
@@ -804,6 +818,7 @@
chrono::milliseconds(1));
EXPECT_GT(chrono::nanoseconds(pi2_connection->monotonic_offset()),
chrono::milliseconds(-1));
+ EXPECT_TRUE(pi2_connection->has_boot_uuid());
}
// Shut everyone else down
@@ -883,6 +898,7 @@
[ping_channel_index, &ping_timestamp_count](const RemoteMessage &header) {
VLOG(1) << "/pi1/aos/remote_timestamps/pi2 RemoteMessage "
<< aos::FlatbufferToJson(&header);
+ EXPECT_TRUE(header.has_boot_uuid());
if (header.channel_index() == ping_channel_index) {
++ping_timestamp_count;
}
@@ -1038,6 +1054,7 @@
[ping_channel_index, &ping_timestamp_count](const RemoteMessage &header) {
VLOG(1) << "/pi1/aos/remote_timestamps/pi2 RemoteMessage "
<< aos::FlatbufferToJson(&header);
+ EXPECT_TRUE(header.has_boot_uuid());
if (header.channel_index() == ping_channel_index) {
++ping_timestamp_count;
}
diff --git a/aos/network/remote_message.fbs b/aos/network/remote_message.fbs
index 5659704..b43f055 100644
--- a/aos/network/remote_message.fbs
+++ b/aos/network/remote_message.fbs
@@ -26,6 +26,9 @@
realtime_remote_time:int64 = -9223372036854775808 (id: 6);
// Queue index of this message on the remote node.
remote_queue_index:uint32 = 4294967295 (id: 7);
+
+ // UUID for this boot.
+ boot_uuid:string (id: 8);
}
root_type RemoteMessage;