Merge "Multiline logger"
diff --git a/aos/events/logging/logfile_utils.cc b/aos/events/logging/logfile_utils.cc
index d455bf5..2cc2f61 100644
--- a/aos/events/logging/logfile_utils.cc
+++ b/aos/events/logging/logfile_utils.cc
@@ -1008,7 +1008,21 @@
for (const std::unique_ptr<SplitMessageReader> &reader :
split_message_readers_) {
- if (CompareFlatBuffer(reader->node(), target_node)) {
+ // In order to identify which logfile(s) map to the target node, do a
+ // logical comparison of the nodes, by confirming that we are either in a
+ // single-node setup (where the nodes will both be nullptr) or that the
+ // node names match (but the other node fields--e.g., hostname lists--may
+ // not).
+ const bool both_null =
+ reader->node() == nullptr && target_node == nullptr;
+ const bool both_have_name =
+ (reader->node() != nullptr) && (target_node != nullptr) &&
+ (reader->node()->has_name() && target_node->has_name());
+ const bool node_names_identical =
+ both_have_name &&
+ (reader->node()->name()->string_view() ==
+ target_node->name()->string_view());
+ if (both_null || node_names_identical) {
if (!found_node) {
found_node = true;
log_file_header_ = CopyFlatBuffer(reader->log_file_header());
diff --git a/aos/events/logging/logger.cc b/aos/events/logging/logger.cc
index de9d344..b3472d7 100644
--- a/aos/events/logging/logger.cc
+++ b/aos/events/logging/logger.cc
@@ -312,9 +312,16 @@
state->channel_merger = std::make_unique<ChannelMerger>(filenames);
} else {
if (replay_configuration) {
- CHECK_EQ(configuration()->nodes()->size(),
+ CHECK_EQ(logged_configuration()->nodes()->size(),
replay_configuration->nodes()->size())
<< ": Log file and replay config need to have matching nodes lists.";
+ for (const Node *node : *logged_configuration()->nodes()) {
+ if (configuration::GetNode(replay_configuration, node) == nullptr) {
+ LOG(FATAL)
+ << "Found node " << FlatbufferToJson(node)
+ << " in logged config that is not present in the replay config.";
+ }
+ }
}
states_.resize(configuration()->nodes()->size());
}
@@ -387,6 +394,11 @@
Register(state->event_loop_unique_ptr.get());
}
+ if (live_nodes_ == 0) {
+ LOG(FATAL)
+ << "Don't have logs from any of the nodes in the replay config--are "
+ "you sure that the replay config matches the original config?";
+ }
// We need to now seed our per-node time offsets and get everything set up to
// run.
diff --git a/aos/events/logging/logger.h b/aos/events/logging/logger.h
index 34dbd24..ce21598 100644
--- a/aos/events/logging/logger.h
+++ b/aos/events/logging/logger.h
@@ -424,6 +424,8 @@
// Returns the offset from the monotonic clock for a node to the distributed
// clock. distributed = monotonic + offset;
std::chrono::nanoseconds offset(int node_index) const {
+ CHECK_LT(node_index, offset_matrix_.rows())
+ << ": Got too high of a node index.";
return -std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::duration<double>(offset_matrix_(node_index))) -
base_offset_matrix_(node_index);
diff --git a/aos/events/logging/logger_test.cc b/aos/events/logging/logger_test.cc
index 9e69ae4..b894bf7 100644
--- a/aos/events/logging/logger_test.cc
+++ b/aos/events/logging/logger_test.cc
@@ -505,6 +505,77 @@
reader.Deregister();
}
+typedef MultinodeLoggerTest MultinodeLoggerDeathTest;
+
+// Test that if we feed the replay with a mismatched node list that we die on
+// the LogReader constructor.
+TEST_F(MultinodeLoggerDeathTest, MultiNodeBadReplayConfig) {
+ const ::std::string tmpdir(getenv("TEST_TMPDIR"));
+ const ::std::string logfile_base = tmpdir + "/multi_logfile";
+ const ::std::string logfile1 = logfile_base + "_pi1_data.bfbs";
+ const ::std::string logfile2 =
+ logfile_base + "_pi2_data/test/aos.examples.Pong.bfbs";
+ const ::std::string logfile3 = logfile_base + "_pi2_data.bfbs";
+
+ // Remove them.
+ unlink(logfile1.c_str());
+ unlink(logfile2.c_str());
+ unlink(logfile3.c_str());
+
+ LOG(INFO) << "Logging data to " << logfile1 << ", " << logfile2 << " and "
+ << logfile3;
+
+ {
+ std::unique_ptr<EventLoop> ping_event_loop =
+ event_loop_factory_.MakeEventLoop("ping", pi1_);
+ Ping ping(ping_event_loop.get());
+ std::unique_ptr<EventLoop> pong_event_loop =
+ event_loop_factory_.MakeEventLoop("pong", pi2_);
+ Pong pong(pong_event_loop.get());
+
+ std::unique_ptr<EventLoop> pi1_logger_event_loop =
+ event_loop_factory_.MakeEventLoop("logger", pi1_);
+ std::unique_ptr<LogNamer> pi1_log_namer =
+ std::make_unique<MultiNodeLogNamer>(
+ logfile_base, pi1_logger_event_loop->configuration(),
+ pi1_logger_event_loop->node());
+
+ std::unique_ptr<EventLoop> pi2_logger_event_loop =
+ event_loop_factory_.MakeEventLoop("logger", pi2_);
+ std::unique_ptr<LogNamer> pi2_log_namer =
+ std::make_unique<MultiNodeLogNamer>(
+ logfile_base, pi2_logger_event_loop->configuration(),
+ pi2_logger_event_loop->node());
+
+ event_loop_factory_.RunFor(chrono::milliseconds(95));
+
+ Logger pi1_logger(std::move(pi1_log_namer), pi1_logger_event_loop.get(),
+ chrono::milliseconds(100));
+
+ Logger pi2_logger(std::move(pi2_log_namer), pi2_logger_event_loop.get(),
+ chrono::milliseconds(100));
+ event_loop_factory_.RunFor(chrono::milliseconds(20000));
+ }
+
+ // Test that, if we add an additional node to the replay config that the
+ // logger complains about the mismatch in number of nodes.
+ FlatbufferDetachedBuffer<Configuration> extra_nodes_config =
+ configuration::MergeWithConfig(&config_.message(), R"({
+ "nodes": [
+ {
+ "name": "extra-node"
+ }
+ ]
+ }
+ )");
+
+ EXPECT_DEATH(LogReader({std::vector<std::string>{logfile1},
+ std::vector<std::string>{logfile3}},
+ &extra_nodes_config.message()),
+ "Log file and replay config need to have matching nodes lists.");
+ ;
+}
+
// Tests that we can read log files where they don't start at the same monotonic
// time.
TEST_F(MultinodeLoggerTest, StaggeredStart) {
diff --git a/aos/ipc_lib/lockless_queue.cc b/aos/ipc_lib/lockless_queue.cc
index f31d80d..ccbe887 100644
--- a/aos/ipc_lib/lockless_queue.cc
+++ b/aos/ipc_lib/lockless_queue.cc
@@ -276,6 +276,50 @@
// Everything should be zero initialized already. So we just need to fill
// everything out properly.
+ // This is the UID we will use for checking signal-sending permission
+ // compatibility.
+ //
+ // The manpage says:
+ // For a process to have permission to send a signal, it must either be
+ // privileged [...], or the real or effective user ID of the sending process
+ // must equal the real or saved set-user-ID of the target process.
+ //
+ // Processes typically initialize a queue in random order as they start up.
+ // This means we need an algorithm for verifying all processes have
+ // permissions to send each other signals which gives the same answer no
+ // matter what order they attach in. We would also like to avoid maintaining a
+ // shared list of the UIDs of all processes.
+ //
+ // To do this while still giving sufficient flexibility for all current use
+ // cases, we track a single UID for the queue. All processes with a matching
+ // euid+suid must have this UID. Any processes with distinct euid/suid must
+ // instead have a matching ruid. This guarantees signals can be sent between
+ // all processes attached to the queue.
+ //
+ // In particular, this allows a process to change only its euid (to interact
+ // with a queue) while still maintaining privileges via its ruid. However, it
+ // can only use privileges in ways that do not require changing the euid back,
+ // because while the euid is different it will not be able to receive signals.
+ // We can't actually verify that, but we can sanity check that things are
+ // valid when the queue is initialized.
+
+ uid_t uid;
+ {
+ uid_t ruid, euid, suid;
+ PCHECK(getresuid(&ruid, &euid, &suid) == 0);
+ // If these are equal, then use them, even if that's different from the real
+ // UID. This allows processes to keep a real UID of 0 (to have permissions
+ // to perform system-level changes) while still being able to communicate
+ // with processes running unprivileged as a distinct user.
+ if (euid == suid) {
+ uid = euid;
+ VLOG(1) << "Using euid==suid " << uid;
+ } else {
+ uid = ruid;
+ VLOG(1) << "Using ruid " << ruid;
+ }
+ }
+
// Grab the mutex. We don't care if the previous reader died. We are going
// to check everything anyways.
GrabQueueSetupLockOrDie grab_queue_setup_lock(memory);
@@ -306,7 +350,7 @@
}
memory->next_queue_index.Invalidate();
- memory->uid = getuid();
+ memory->uid = uid;
for (size_t i = 0; i < memory->num_senders(); ++i) {
::aos::ipc_lib::Sender *s = memory->GetSender(i);
@@ -321,7 +365,7 @@
// redo initialization.
memory->initialized = true;
} else {
- CHECK_EQ(getuid(), memory->uid) << ": UIDs must match for all processes";
+ CHECK_EQ(uid, memory->uid) << ": UIDs must match for all processes";
}
return memory;
diff --git a/y2020/constants.cc b/y2020/constants.cc
index eefc158..131b0de 100644
--- a/y2020/constants.cc
+++ b/y2020/constants.cc
@@ -26,6 +26,7 @@
const uint16_t kCompTeamNumber = 971;
const uint16_t kPracticeTeamNumber = 9971;
+const uint16_t kSpareRoborioTeamNumber = 6971;
const Values *DoGetValuesForTeam(uint16_t team) {
Values *const r = new Values();
@@ -99,6 +100,7 @@
switch (team) {
// A set of constants for tests.
case 1:
+ case kSpareRoborioTeamNumber:
break;
case kCompTeamNumber:
diff --git a/y2020/y2020_roborio.json b/y2020/y2020_roborio.json
index 7de7f51..b63982d 100644
--- a/y2020/y2020_roborio.json
+++ b/y2020/y2020_roborio.json
@@ -263,6 +263,7 @@
"hostname": "roborio",
"hostnames": [
"roboRIO-971-FRC",
+ "roboRIO-6971-FRC",
"roboRIO-7971-FRC",
"roboRIO-8971-FRC",
"roboRIO-9971-FRC"