Add support for reading, merging, and querying configs
Change-Id: I04a07cf5e9a6b41213b3101f3e04be350e50b41f
diff --git a/aos/configuration.cc b/aos/configuration.cc
index 94606ec..a1cc614 100644
--- a/aos/configuration.cc
+++ b/aos/configuration.cc
@@ -8,11 +8,59 @@
#include <ifaddrs.h>
#include <unistd.h>
-#include "aos/logging/logging.h"
-#include "aos/unique_malloc_ptr.h"
+#include "absl/container/btree_set.h"
+#include "absl/strings/string_view.h"
+#include "aos/configuration_generated.h"
+#include "aos/flatbuffer_merge.h"
+#include "aos/json_to_flatbuffer.h"
#include "aos/once.h"
+#include "aos/unique_malloc_ptr.h"
+#include "aos/util/file.h"
+#include "glog/logging.h"
namespace aos {
+
+// Define the compare and equal operators for Location and Application so we can
+// insert them in the btree below.
+//
+// These are not in headers because they are only comparing part of the
+// flatbuffer, and it seems weird to expose that as *the* compare operator. And
+// I can't put them in an anonymous namespace because they wouldn't be found
+// that way by the btree.
+bool operator<(const Flatbuffer<Location> &lhs,
+ const Flatbuffer<Location> &rhs) {
+ int name_compare = lhs.message().name()->string_view().compare(
+ rhs.message().name()->string_view());
+ if (name_compare == 0) {
+ return lhs.message().type()->string_view() <
+ rhs.message().type()->string_view();
+ } else if (name_compare < 0) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool operator==(const Flatbuffer<Location> &lhs,
+ const Flatbuffer<Location> &rhs) {
+ return lhs.message().name()->string_view() ==
+ rhs.message().name()->string_view() &&
+ lhs.message().type()->string_view() ==
+ rhs.message().type()->string_view();
+}
+
+bool operator==(const Flatbuffer<Application> &lhs,
+ const Flatbuffer<Application> &rhs) {
+ return lhs.message().name()->string_view() ==
+ rhs.message().name()->string_view();
+}
+
+bool operator<(const Flatbuffer<Application> &lhs,
+ const Flatbuffer<Application> &rhs) {
+ return lhs.message().name()->string_view() <
+ rhs.message().name()->string_view();
+}
+
namespace configuration {
namespace {
@@ -23,21 +71,21 @@
static const char *kOverrideVariable = "FRC971_IP_OVERRIDE";
const char *override_ip = getenv(kOverrideVariable);
if (override_ip != NULL) {
- LOG(INFO, "Override IP is %s\n", override_ip);
+ LOG(INFO) << "Override IP is " << override_ip;
static in_addr r;
if (inet_aton(override_ip, &r) != 0) {
return &r;
} else {
- LOG(WARNING, "error parsing %s value '%s'\n",
- kOverrideVariable, override_ip);
+ LOG(WARNING) << "error parsing " << kOverrideVariable << " value '"
+ << override_ip << "'";
}
} else {
- LOG(INFO, "Couldn't get environmental variable.\n");
+ LOG(INFO) << "Couldn't get environmental variable.";
}
ifaddrs *addrs;
if (getifaddrs(&addrs) != 0) {
- PLOG(FATAL, "getifaddrs(%p) failed", &addrs);
+ PLOG(FATAL) << "getifaddrs(" << &addrs << ") failed";
}
// Smart pointers don't work very well for iterating through a linked list,
// but it does do a very nice job of making sure that addrs gets freed.
@@ -54,8 +102,9 @@
}
}
}
- LOG(FATAL, "couldn't find an AF_INET interface named \"%s\"\n",
- kLinuxNetInterface);
+ LOG(FATAL) << "couldn't find an AF_INET interface named \""
+ << kLinuxNetInterface << "\"";
+ return nullptr;
}
const char *DoGetRootDirectory() {
@@ -69,18 +118,19 @@
ssize_t ret = readlink("/proc/self/exe", r, size);
if (ret < 0) {
if (ret != -1) {
- LOG(WARNING, "it returned %zd, not -1\n", ret);
+ LOG(WARNING) << "it returned " << ret << ", not -1";
}
- PLOG(FATAL, "readlink(\"/proc/self/exe\", %p, %zu) failed", r, size);
+ PLOG(FATAL) << "readlink(\"/proc/self/exe\", " << r << ", " << size
+ << ") failed";
}
if (ret < size) {
void *last_slash = memrchr(r, '/', ret);
if (last_slash == NULL) {
r[ret] = '\0';
- LOG(FATAL, "couldn't find a '/' in \"%s\"\n", r);
+ LOG(FATAL) << "couldn't find a '/' in \"" << r << "\"";
}
*static_cast<char *>(last_slash) = '\0';
- LOG(INFO, "got a root dir of \"%s\"\n", r);
+ LOG(INFO) << "got a root dir of \"" << r << "\"";
return r;
}
}
@@ -95,6 +145,225 @@
return r;
}
+// Extracts the folder part of a path. Returns ./ if there is no path.
+absl::string_view ExtractFolder(const absl::string_view filename) {
+ auto last_slash_pos = filename.find_last_of("/\\");
+
+ return last_slash_pos == absl::string_view::npos
+ ? absl::string_view("./")
+ : filename.substr(0, last_slash_pos + 1);
+}
+
+Flatbuffer<Configuration> ReadConfig(
+ const absl::string_view path, absl::btree_set<std::string> *visited_paths) {
+ Flatbuffer<Configuration> config(JsonToFlatbuffer(
+ util::ReadFileToStringOrDie(path), ConfigurationTypeTable()));
+ // Depth first. Take the following example:
+ //
+ // config1.json:
+ // {
+ // "locations": [
+ // {
+ // "name": "/foo",
+ // "type": ".aos.bar",
+ // "max_size": 5
+ // }
+ // ],
+ // "imports": [
+ // "config2.json",
+ // ]
+ // }
+ //
+ // config2.json:
+ // {
+ // "locations": [
+ // {
+ // "name": "/foo",
+ // "type": ".aos.bar",
+ // "max_size": 7
+ // }
+ // ],
+ // }
+ //
+ // We want the main config (config1.json) to be able to override the imported
+ // config. That means that it needs to be merged into the imported configs,
+ // not the other way around.
+
+ // Track that we have seen this file before recursing.
+ visited_paths->insert(::std::string(path));
+
+ if (config.message().has_imports()) {
+ // Capture the imports.
+ const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *v =
+ config.message().imports();
+
+ // And then wipe them. This gets GCed when we merge later.
+ config.mutable_message()->clear_imports();
+
+ // Start with an empty configuration to merge into.
+ Flatbuffer<Configuration> merged_config =
+ Flatbuffer<Configuration>::Empty();
+
+ const ::std::string folder(ExtractFolder(path));
+
+ for (const flatbuffers::String *str : *v) {
+ const ::std::string included_config = folder + str->c_str();
+ // Abort on any paths we have already seen.
+ CHECK(visited_paths->find(included_config) == visited_paths->end())
+ << ": Found duplicate file " << included_config << " while reading "
+ << path;
+
+ // And them merge everything in.
+ merged_config = MergeFlatBuffers(
+ merged_config, ReadConfig(included_config, visited_paths));
+ }
+
+ // Finally, merge this file in.
+ config = MergeFlatBuffers(merged_config, config);
+ }
+ return config;
+}
+
+// Remove duplicate entries, and handle overrides.
+Flatbuffer<Configuration> MergeConfiguration(
+ const Flatbuffer<Configuration> &config) {
+ // Store all the locations in a sorted set. This lets us track locations we
+ // have seen before and merge the updates in.
+ absl::btree_set<Flatbuffer<Location>> locations;
+
+ if (config.message().has_locations()) {
+ for (const Location *l : *config.message().locations()) {
+ // Ignore malformed entries.
+ if (!l->has_name()) {
+ continue;
+ }
+ if (!l->has_type()) {
+ continue;
+ }
+
+ // Attempt to insert the location.
+ auto result = locations.insert(CopyFlatBuffer(l));
+ if (!result.second) {
+ // Already there, so merge the new table into the original.
+ *result.first = MergeFlatBuffers(*result.first, CopyFlatBuffer(l));
+ }
+ }
+ }
+
+ // Now repeat this for the application list.
+ absl::btree_set<Flatbuffer<Application>> applications;
+ if (config.message().has_applications()) {
+ for (const Application *a : *config.message().applications()) {
+ if (!a->has_name()) {
+ continue;
+ }
+
+ auto result = applications.insert(CopyFlatBuffer(a));
+ if (!result.second) {
+ *result.first = MergeFlatBuffers(*result.first, CopyFlatBuffer(a));
+ }
+ }
+ }
+
+ flatbuffers::FlatBufferBuilder fbb;
+ fbb.ForceDefaults(1);
+
+ // Start by building the vectors. They need to come before the final table.
+ // Locations
+ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Location>>>
+ locations_offset;
+ {
+ ::std::vector<flatbuffers::Offset<Location>> location_offsets;
+ for (const Flatbuffer<Location> &l : locations) {
+ location_offsets.emplace_back(
+ CopyFlatBuffer<Location>(&l.message(), &fbb));
+ }
+ locations_offset = fbb.CreateVector(location_offsets);
+ }
+
+ // Applications
+ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Application>>>
+ applications_offset;
+ {
+ ::std::vector<flatbuffers::Offset<Application>> applications_offsets;
+ for (const Flatbuffer<Application> &a : applications) {
+ applications_offsets.emplace_back(
+ CopyFlatBuffer<Application>(&a.message(), &fbb));
+ }
+ applications_offset = fbb.CreateVector(applications_offsets);
+ }
+
+ // Just copy the maps
+ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Map>>>
+ maps_offset;
+ {
+ ::std::vector<flatbuffers::Offset<Map>> map_offsets;
+ if (config.message().has_maps()) {
+ for (const Map *m : *config.message().maps()) {
+ map_offsets.emplace_back(CopyFlatBuffer<Map>(m, &fbb));
+ }
+ maps_offset = fbb.CreateVector(map_offsets);
+ }
+ }
+
+ // And then build a Configuration with them all.
+ ConfigurationBuilder configuration_builder(fbb);
+ configuration_builder.add_locations(locations_offset);
+ if (config.message().has_maps()) {
+ configuration_builder.add_maps(maps_offset);
+ }
+ configuration_builder.add_applications(applications_offset);
+
+ fbb.Finish(configuration_builder.Finish());
+ return fbb.Release();
+}
+
+// Compares (l < p) a location, and a name, type tuple.
+bool CompareLocations(const Location *l,
+ ::std::pair<absl::string_view, absl::string_view> p) {
+ int name_compare = l->name()->string_view().compare(p.first);
+ if (name_compare == 0) {
+ return l->type()->string_view() < p.second;
+ } else if (name_compare < 0) {
+ return true;
+ } else {
+ return false;
+ }
+};
+
+// Compares for equality (l == p) a location, and a name, type tuple.
+bool EqualsLocations(const Location *l,
+ ::std::pair<absl::string_view, absl::string_view> p) {
+ return l->name()->string_view() == p.first &&
+ l->type()->string_view() == p.second;
+}
+
+// Compares (l < p) an application, and a name;
+bool CompareApplications(const Application *a, absl::string_view name) {
+ return a->name()->string_view() < name;
+};
+
+// Compares for equality (l == p) an application, and a name;
+bool EqualsApplications(const Application *a, absl::string_view name) {
+ return a->name()->string_view() == name;
+}
+
+// Maps name for the provided maps. Modifies name.
+void HandleMaps(const flatbuffers::Vector<flatbuffers::Offset<aos::Map>> *maps,
+ absl::string_view *name) {
+ // For the same reason we merge configs in reverse order, we want to process
+ // maps in reverse order. That lets the outer config overwrite locations from
+ // the inner configs.
+ for (auto i = maps->rbegin(); i != maps->rend(); ++i) {
+ if (i->has_match() && i->match()->has_name() && i->has_rename() &&
+ i->rename()->has_name() && i->match()->name()->string_view() == *name) {
+ VLOG(1) << "Renamed \"" << *name << "\" to \""
+ << i->rename()->name()->string_view() << "\"";
+ *name = i->rename()->name()->string_view();
+ }
+ }
+}
+
} // namespace
const char *GetRootDirectory() {
@@ -112,5 +381,55 @@
return *once.Get();
}
+Flatbuffer<Configuration> ReadConfig(const absl::string_view path) {
+ // We only want to read a file once. So track the visited files in a set.
+ absl::btree_set<std::string> visited_paths;
+ return MergeConfiguration(ReadConfig(path, &visited_paths));
+}
+
+const Location *GetLocation(const Flatbuffer<Configuration> &config,
+ absl::string_view name, absl::string_view type,
+ absl::string_view application_name) {
+ VLOG(1) << "Looking up { \"name\": \"" << name << "\", \"type\": \"" << type
+ << "\" }";
+
+ // First handle application specific maps. Only do this if we have a matching
+ // application name, and it has maps.
+ if (config.message().has_applications()) {
+ auto application_iterator =
+ std::lower_bound(config.message().applications()->cbegin(),
+ config.message().applications()->cend(),
+ application_name, CompareApplications);
+ if (application_iterator != config.message().applications()->cend() &&
+ EqualsApplications(*application_iterator, application_name)) {
+ if (application_iterator->has_maps()) {
+ HandleMaps(application_iterator->maps(), &name);
+ }
+ }
+ }
+
+ // Now do global maps.
+ if (config.message().has_maps()) {
+ HandleMaps(config.message().maps(), &name);
+ }
+
+ // Then look for the location.
+ auto location_iterator =
+ std::lower_bound(config.message().locations()->cbegin(),
+ config.message().locations()->cend(),
+ std::make_pair(name, type), CompareLocations);
+
+ // Make sure we actually found it, and it matches.
+ if (location_iterator != config.message().locations()->cend() &&
+ EqualsLocations(*location_iterator, std::make_pair(name, type))) {
+ VLOG(1) << "Found: " << FlatbufferToJson(*location_iterator);
+ return *location_iterator;
+ } else {
+ VLOG(1) << "No match for { \"name\": \"" << name << "\", \"type\": \""
+ << type << "\" }";
+ return nullptr;
+ }
+}
+
} // namespace configuration
} // namespace aos