Add basic smoke test for MCAP converter
Grab the latest release of the "mcap" tool, which
includes a validator for MCAP files. Then
write a simple test that generates an AOS log,
converts it to MCAP, and tests that it is correct.
Change-Id: Ia2befa535405de1110810706b76f48782064da32
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/WORKSPACE b/WORKSPACE
index 2113efe..df7ed47 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1175,3 +1175,10 @@
name = "com_github_nlohmann_json",
path = "third_party/com_github_nlohmann_json",
)
+
+http_file(
+ name = "com_github_foxglove_mcap_mcap",
+ executable = True,
+ sha256 = "cf4dfcf71e20a60406aaded03a165312c1ca535b509ead90eb1846fc598137d2",
+ urls = ["https://github.com/foxglove/mcap/releases/download/releases%2Fmcap-cli%2Fv0.0.5/mcap-linux-amd64"],
+)
diff --git a/aos/util/BUILD b/aos/util/BUILD
index 8d21c47..12e5ab7 100644
--- a/aos/util/BUILD
+++ b/aos/util/BUILD
@@ -61,6 +61,40 @@
],
)
+cc_binary(
+ name = "generate_test_log",
+ testonly = True,
+ srcs = ["generate_test_log.cc"],
+ data = ["//aos/events:pingpong_config"],
+ deps = [
+ "//aos:configuration",
+ "//aos/events:ping_lib",
+ "//aos/events:pong_lib",
+ "//aos/events:simulated_event_loop",
+ "//aos/events/logging:log_writer",
+ "//aos/testing:path",
+ ],
+)
+
+py_test(
+ name = "log_to_mcap_test",
+ srcs = ["log_to_mcap_test.py"],
+ args = [
+ "--log_to_mcap",
+ "$(location :log_to_mcap)",
+ "--mcap",
+ "$(location @com_github_foxglove_mcap_mcap//file)",
+ "--generate_log",
+ "$(location :generate_test_log)",
+ ],
+ data = [
+ ":generate_test_log",
+ ":log_to_mcap",
+ "@com_github_foxglove_mcap_mcap//file",
+ ],
+ target_compatible_with = ["@platforms//cpu:x86_64"],
+)
+
cc_test(
name = "mcap_logger_test",
srcs = ["mcap_logger_test.cc"],
diff --git a/aos/util/generate_test_log.cc b/aos/util/generate_test_log.cc
new file mode 100644
index 0000000..7338af8
--- /dev/null
+++ b/aos/util/generate_test_log.cc
@@ -0,0 +1,38 @@
+#include "aos/configuration.h"
+#include "aos/events/logging/log_writer.h"
+#include "aos/init.h"
+#include "aos/json_to_flatbuffer.h"
+#include "gflags/gflags.h"
+#include "aos/testing/path.h"
+#include "aos/events/ping_lib.h"
+#include "aos/events/pong_lib.h"
+
+DEFINE_string(output_folder, "", "Name of folder to write the generated logfile to.");
+
+int main(int argc, char **argv) {
+ aos::InitGoogle(&argc, &argv);
+
+ const aos::FlatbufferDetachedBuffer<aos::Configuration> config =
+ aos::configuration::ReadConfig(
+ aos::testing::ArtifactPath("aos/events/pingpong_config.json"));
+
+ aos::SimulatedEventLoopFactory event_loop_factory(&config.message());
+
+ // Event loop and app for Ping
+ std::unique_ptr<aos::EventLoop> ping_event_loop =
+ event_loop_factory.MakeEventLoop("ping");
+ aos::Ping ping(ping_event_loop.get());
+
+ // Event loop and app for Pong
+ std::unique_ptr<aos::EventLoop> pong_event_loop =
+ event_loop_factory.MakeEventLoop("pong");
+ aos::Pong pong(pong_event_loop.get());
+
+ std::unique_ptr<aos::EventLoop> log_writer_event_loop =
+ event_loop_factory.MakeEventLoop("log_writer");
+ aos::logger::Logger writer(log_writer_event_loop.get());
+ writer.StartLoggingOnRun(FLAGS_output_folder);
+
+ event_loop_factory.RunFor(std::chrono::seconds(10));
+ return 0;
+}
diff --git a/aos/util/log_to_mcap.cc b/aos/util/log_to_mcap.cc
index 17555a4..49aca69 100644
--- a/aos/util/log_to_mcap.cc
+++ b/aos/util/log_to_mcap.cc
@@ -24,8 +24,9 @@
reader.Register();
const aos::Node *node =
- aos::configuration::GetNode(reader.configuration(), FLAGS_node);
- CHECK_NOTNULL(node);
+ FLAGS_node.empty()
+ ? nullptr
+ : aos::configuration::GetNode(reader.configuration(), FLAGS_node);
std::unique_ptr<aos::EventLoop> mcap_event_loop =
reader.event_loop_factory()->MakeEventLoop("mcap", node);
CHECK(!FLAGS_output_path.empty());
diff --git a/aos/util/log_to_mcap_test.py b/aos/util/log_to_mcap_test.py
new file mode 100644
index 0000000..86c3d1d
--- /dev/null
+++ b/aos/util/log_to_mcap_test.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# This script is meant to act as a test to confirm that our log_to_mcap converter produces
+# a valid MCAP file. To do so, it first generates an AOS log, then converts it to MCAP, and
+# then runs the "mcap doctor" tool on it to confirm compliance with the standard.
+import argparse
+import subprocess
+import sys
+import tempfile
+import time
+from typing import Sequence, Text
+
+
+def main(argv: Sequence[Text]):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--log_to_mcap", required=True, help="Path to log_to_mcap binary.")
+ parser.add_argument("--mcap", required=True, help="Path to mcap binary.")
+ parser.add_argument("--generate_log", required=True, help="Path to logfile generator.")
+ args = parser.parse_args(argv)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ log_name = tmpdir + "/test_log/"
+ mcap_name = tmpdir + "/log.mcap"
+ subprocess.run([args.generate_log, "--output_folder", log_name]).check_returncode()
+ # Run with a really small chunk size, to force a multi-chunk file.
+ subprocess.run(
+ [args.log_to_mcap, "--output_path", mcap_name, "--mcap_chunk_size", "1000",
+ log_name]).check_returncode()
+ # MCAP attempts to find $HOME/.mcap.yaml, and dies on $HOME not existing. So
+ # give it an arbitrary config location (it seems to be fine with a non-existent config).
+ doctor_result = subprocess.run(
+ [args.mcap, "doctor", mcap_name, "--config", tmpdir + "/.mcap.yaml"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding='utf-8')
+ print(doctor_result.stdout)
+ print(doctor_result.stderr)
+ # mcap doctor doesn't actually return a non-zero exit code on certain failures...
+ # See https://github.com/foxglove/mcap/issues/356
+ if len(doctor_result.stderr) != 0:
+ print("Didn't expect any stderr output.")
+ return 1
+ if doctor_result.stdout != f"Examining {mcap_name}\n":
+ print("Only expected one line of stdout.")
+ return 1
+ doctor_result.check_returncode()
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/aos/util/mcap_logger.cc b/aos/util/mcap_logger.cc
index 1cb9c32..f4056ea 100644
--- a/aos/util/mcap_logger.cc
+++ b/aos/util/mcap_logger.cc
@@ -3,11 +3,10 @@
#include "absl/strings/str_replace.h"
#include "single_include/nlohmann/json.hpp"
+DEFINE_uint64(mcap_chunk_size, 10000000,
+ "Size, in bytes, of individual MCAP chunks");
+
namespace aos {
-namespace {
-// Amount of data to allow in each chunk before creating a new chunk.
-constexpr size_t kChunkSize = 10000000;
-}
nlohmann::json JsonSchemaForFlatbuffer(const FlatbufferType &type,
JsonSchemaRecursion recursion_level) {
@@ -137,7 +136,8 @@
event_loop_->MakeRawWatcher(
channel, [this, id, channel](const Context &context, const void *) {
WriteMessage(id, channel, context, ¤t_chunk_);
- if (static_cast<uint64_t>(current_chunk_.tellp()) > kChunkSize) {
+ if (static_cast<uint64_t>(current_chunk_.tellp()) >
+ FLAGS_mcap_chunk_size) {
WriteChunk();
}
});