Only produce full aos_config json on demand
These files are massive (100 MB each), and we are sending them across
the network to remote workers.
Change-Id: Ied70998558360c8dda86657af83b13235efa22b6
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/aos/config.bzl b/aos/config.bzl
index d281027..faaf45a 100644
--- a/aos/config.bzl
+++ b/aos/config.bzl
@@ -41,22 +41,32 @@
all_files = flatbuffers_depset.to_list() + src_depset.to_list()
ctx.actions.run(
- outputs = [config, stripped_config, binary_config],
+ outputs = [stripped_config, binary_config],
inputs = all_files,
arguments = ctx.attr.flags + [
- config.path,
- stripped_config.path,
- binary_config.path,
+ "--stripped_output=" + stripped_config.path,
+ "--binary_output=" + binary_config.path,
(ctx.label.workspace_root or ".") + "/" + ctx.files.src[0].short_path,
ctx.bin_dir.path,
] + [f.path for f in flatbuffers_depset.to_list()],
progress_message = "Flattening config",
executable = ctx.executable._config_flattener,
)
- runfiles = ctx.runfiles(files = [config, stripped_config, binary_config])
+ ctx.actions.run(
+ outputs = [config],
+ inputs = all_files,
+ arguments = ctx.attr.flags + [
+ "--full_output=" + config.path,
+ (ctx.label.workspace_root or ".") + "/" + ctx.files.src[0].short_path,
+ ctx.bin_dir.path,
+ ] + [f.path for f in flatbuffers_depset.to_list()],
+ progress_message = "Flattening config",
+ executable = ctx.executable._config_flattener,
+ )
+ runfiles = ctx.runfiles(files = [stripped_config, binary_config])
return [
DefaultInfo(
- files = depset([config, stripped_config, binary_config]),
+ files = depset([stripped_config, binary_config]),
runfiles = runfiles,
),
AosConfigInfo(
diff --git a/aos/config_flattener.cc b/aos/config_flattener.cc
index b089d24..8170a89 100644
--- a/aos/config_flattener.cc
+++ b/aos/config_flattener.cc
@@ -9,19 +9,25 @@
#include "aos/json_to_flatbuffer.h"
#include "aos/util/file.h"
+DEFINE_string(full_output, "",
+ "If provided, the path to write the full json configuration to.");
+DEFINE_string(
+ stripped_output, "",
+ "If provided, the path to write the stripped json configuration to.");
+DEFINE_string(
+ binary_output, "",
+ "If provided, the path to write the binary flatbuffer configuration to.");
+
namespace aos {
int Main(int argc, char **argv) {
- CHECK_GE(argc, 6) << ": Too few arguments";
+ CHECK_GE(argc, 3) << ": Too few arguments";
- const char *full_output = argv[1];
- const char *stripped_output = argv[2];
- const char *binary_output = argv[3];
- const char *config_path = argv[4];
+ const char *config_path = argv[1];
// In order to support not only importing things by absolute path, but also
// importing the outputs of genrules (rather than just manually written
// files), we need to tell ReadConfig where the generated files directory is.
- const char *bazel_outs_directory = argv[5];
+ const char *bazel_outs_directory = argv[2];
VLOG(1) << "Reading " << config_path;
FlatbufferDetachedBuffer<Configuration> config =
@@ -38,7 +44,7 @@
std::vector<aos::FlatbufferVector<reflection::Schema>> schemas;
- for (int i = 6; i < argc; ++i) {
+ for (int i = 3; i < argc; ++i) {
schemas.emplace_back(FileToFlatbuffer<reflection::Schema>(argv[i]));
}
@@ -55,11 +61,17 @@
// TODO(austin): Figure out how to squash the schemas onto 1 line so it is
// easier to read?
VLOG(1) << "Flattened config is " << merged_config_json;
- util::WriteStringToFileOrDie(full_output, merged_config_json);
- util::WriteStringToFileOrDie(
- stripped_output,
- FlatbufferToJson(&config.message(), {.multi_line = true}));
- aos::WriteFlatbufferToFile(binary_output, merged_config);
+ if (!FLAGS_full_output.empty()) {
+ util::WriteStringToFileOrDie(FLAGS_full_output, merged_config_json);
+ }
+ if (!FLAGS_stripped_output.empty()) {
+ util::WriteStringToFileOrDie(
+ FLAGS_stripped_output,
+ FlatbufferToJson(&config.message(), {.multi_line = true}));
+ }
+ if (!FLAGS_binary_output.empty()) {
+ aos::WriteFlatbufferToFile(FLAGS_binary_output, merged_config);
+ }
return 0;
}