Use starterd to pre-allocate all AOS message queue shared memory.
AOS uses shared memory for intranode message channels. The first
application that creates a sender/fetcher/watcher for each channel
allocates the shared memory for the message queues, and is given credit
for the shared memory as part of its memory footprint.
The non-deterministic launch order of all the shasta applications means
that the measured memory footprint of many of the processes can vary
greatly, depending on where it was in the launch order that time.
As a result, the memory_limit values in the config files are way
bigger than they need to be, if we could have deterministic measurements
of the actual memory usage of each process.
The starterd process is what actually spawns all the processes, using
the configuration data. Making this process create MemoryMappedQueues
for every intranode channel on its node will pre-allocate all the
message queue shared memory, which dramatially reduces the measured
memory usage of most of the processes it spawns, and makes those numbers
much more deterministic.
Renamed MMappedQueue to MemoryMappedQueue and moved it from
shm_event_loop.cc to new source files in the lockless_queue library.
Update starterd_lib to identify all channels in the config that are
readable on this node and allocate MemoryMappedQueues for each.
Change-Id: I40649b61653968495ba616b72f95d799a8e06414
Signed-off-by: Brian J Griglak <brian.griglak@bluerivertech.com>
Signed-off-by: Austin Schuh <austin.schuh@bluerivertech.com>
diff --git a/aos/starter/starterd_lib.cc b/aos/starter/starterd_lib.cc
index 84e4d00..485d1f1 100644
--- a/aos/starter/starterd_lib.cc
+++ b/aos/starter/starterd_lib.cc
@@ -8,6 +8,13 @@
#include "glog/logging.h"
#include "glog/stl_logging.h"
+// FLAGS_shm_base is defined elsewhere, declare it here so it can be used
+// to override the shared memory folder for unit testing.
+DECLARE_string(shm_base);
+// FLAGS_permissions is defined elsewhere, declare it here so it can be used
+// to set the file permissions on the shared memory block.
+DECLARE_uint32(permissions);
+
namespace aos {
namespace starter {
@@ -32,6 +39,7 @@
max_status_count_(
event_loop_.GetChannel<aos::starter::Status>("/aos")->frequency() -
1),
+ shm_base_(FLAGS_shm_base),
listener_(&event_loop_,
[this](signalfd_siginfo signal) { OnSignal(signal); }),
top_(&event_loop_) {
@@ -67,6 +75,8 @@
}
}
+ // Catalogue all the applications for this node, so we can keep an eye on
+ // them.
if (config_msg_->has_applications()) {
const flatbuffers::Vector<flatbuffers::Offset<aos::Application>>
*applications = config_msg_->applications();
@@ -88,6 +98,19 @@
}
}
}
+
+ // Catalogue all the intranode channels for this node, and create
+ // MemoryMappedQueues for each one to allocate the shared memory before
+ // spawning any shasta process.
+ if (config_msg_->has_channels()) {
+ const aos::Node *this_node = event_loop_.node();
+ std::vector<const aos::Channel *> intranode_channels;
+ for (const aos::Channel *channel : *config_msg_->channels()) {
+ if (aos::configuration::ChannelIsReadableOnNode(channel, this_node)) {
+ AddChannel(channel);
+ }
+ }
+ }
}
void Starter::HandleStarterRpc(const StarterRpc &command) {
@@ -220,5 +243,14 @@
builder.CheckOk(builder.Send(status_builder.Finish()));
}
+void Starter::AddChannel(const aos::Channel *channel) {
+ CHECK_NOTNULL(channel);
+ shm_queues_.emplace_back(std::make_unique<aos::ipc_lib::MemoryMappedQueue>(
+ shm_base_, FLAGS_permissions, event_loop_.configuration(), channel));
+ VLOG(1) << "Created MemoryMappedQueue for "
+ << aos::configuration::StrippedChannelToString(channel) << " under "
+ << shm_base_;
+}
+
} // namespace starter
} // namespace aos