Merge "Use starterd to pre-allocate all AOS message queue shared memory."
diff --git a/aos/BUILD b/aos/BUILD
index 69303a9..d78b303 100644
--- a/aos/BUILD
+++ b/aos/BUILD
@@ -542,6 +542,7 @@
         ":configuration",
         "//aos:init",
         "//aos/events:shm_event_loop",
+        "//aos/events:simulated_event_loop",
         "@com_github_google_glog//:glog",
     ],
 )
diff --git a/aos/aos_cli_utils.cc b/aos/aos_cli_utils.cc
index dbd1bcd..2016a71 100644
--- a/aos/aos_cli_utils.cc
+++ b/aos/aos_cli_utils.cc
@@ -4,8 +4,14 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <chrono>
 #include <iostream>
 
+#include "aos/configuration.h"
+#include "aos/events/shm_event_loop.h"
+#include "aos/events/simulated_event_loop.h"
+#include "aos/time/time.h"
+
 DEFINE_string(config, "aos_config.json", "File path of aos configuration");
 
 DEFINE_bool(
@@ -21,6 +27,8 @@
 namespace aos {
 namespace {
 
+namespace chrono = std::chrono;
+
 bool EndsWith(std::string_view str, std::string_view ending) {
   const std::size_t offset = str.size() - ending.size();
   return str.size() >= ending.size() &&
@@ -28,6 +36,27 @@
                     ending.end());
 }
 
+void StreamSeconds(std::ostream &stream,
+                   const aos::monotonic_clock::time_point now) {
+  if (now < monotonic_clock::epoch()) {
+    chrono::seconds seconds =
+        chrono::duration_cast<chrono::seconds>(now.time_since_epoch());
+
+    stream << "-" << -seconds.count() << "." << std::setfill('0')
+           << std::setw(9)
+           << chrono::duration_cast<chrono::nanoseconds>(seconds -
+                                                         now.time_since_epoch())
+                  .count();
+  } else {
+    chrono::seconds seconds =
+        chrono::duration_cast<chrono::seconds>(now.time_since_epoch());
+    stream << seconds.count() << "." << std::setfill('0') << std::setw(9)
+           << chrono::duration_cast<chrono::nanoseconds>(
+                  now.time_since_epoch() - seconds)
+                  .count();
+  }
+}
+
 }  // namespace
 
 bool CliUtilInfo::Initialize(
@@ -179,4 +208,87 @@
   std::cout << ')';
 }
 
+void PrintMessage(const std::string_view node_name, const aos::Channel *channel,
+                  const aos::Context &context, aos::FastStringBuilder *builder,
+                  PrintOptions options) {
+  // Print the flatbuffer out to stdout, both to remove the
+  // unnecessary cruft from glog and to allow the user to readily
+  // redirect just the logged output independent of any debugging
+  // information on stderr.
+
+  builder->Reset();
+
+  CHECK(flatbuffers::Verify(*channel->schema(),
+                            *channel->schema()->root_table(),
+                            static_cast<const uint8_t *>(context.data),
+                            static_cast<size_t>(context.size)))
+      << ": Corrupted flatbuffer on " << channel->name()->c_str() << " "
+      << channel->type()->c_str();
+
+  aos::FlatbufferToJson(
+      builder, channel->schema(), static_cast<const uint8_t *>(context.data),
+      {options.pretty, static_cast<size_t>(options.max_vector_size),
+       options.pretty_max, options.use_hex});
+
+  if (options.json) {
+    std::cout << "{";
+    if (!node_name.empty()) {
+      std::cout << "\"node\": \"" << node_name << "\", ";
+    }
+    std::cout << "\"monotonic_event_time\": ";
+    StreamSeconds(std::cout, context.monotonic_event_time);
+    std::cout << ", \"realtime_event_time\": \"" << context.realtime_event_time
+              << "\", ";
+
+    if (context.monotonic_remote_time != context.monotonic_event_time) {
+      std::cout << "\"monotonic_remote_time\": ";
+      StreamSeconds(std::cout, context.monotonic_remote_time);
+      std::cout << ", \"realtime_remote_time\": \""
+                << context.realtime_remote_time << "\", ";
+    }
+
+    std::cout << "\"channel\": "
+              << aos::configuration::StrippedChannelToString(channel)
+              << ", \"data\": " << *builder << "}\n";
+  } else {
+    if (!node_name.empty()) {
+      std::cout << node_name << " ";
+    }
+
+    if (options.print_timestamps) {
+      if (context.monotonic_remote_time != context.monotonic_event_time) {
+        std::cout << context.realtime_event_time << " ("
+                  << context.monotonic_event_time << ") sent "
+                  << context.realtime_remote_time << " ("
+                  << context.monotonic_remote_time << ") "
+                  << channel->name()->c_str() << ' ' << channel->type()->c_str()
+                  << ": " << *builder << "\n";
+      } else {
+        std::cout << context.realtime_event_time << " ("
+                  << context.monotonic_event_time << ") "
+                  << channel->name()->c_str() << ' ' << channel->type()->c_str()
+                  << ": " << *builder << "\n";
+      }
+    } else {
+      std::cout << *builder << '\n';
+    }
+  }
+}
+
+void PrintMessage(const aos::Channel *channel, const aos::Context &context,
+                  aos::FastStringBuilder *builder, PrintOptions options) {
+  PrintMessage("", channel, context, builder, options);
+}
+
+void PrintMessage(const std::string_view node_name,
+                  aos::NodeEventLoopFactory *node_factory,
+                  const aos::Channel *channel, const aos::Context &context,
+                  aos::FastStringBuilder *builder, PrintOptions options) {
+  if (!options.json && options.distributed_clock) {
+    std::cout << node_factory->ToDistributedClock(context.monotonic_event_time)
+              << " ";
+  }
+  PrintMessage(node_name, channel, context, builder, options);
+}
+
 }  // namespace aos
diff --git a/aos/aos_cli_utils.h b/aos/aos_cli_utils.h
index a2d8c62..7e143dd 100644
--- a/aos/aos_cli_utils.h
+++ b/aos/aos_cli_utils.h
@@ -3,10 +3,40 @@
 
 #include "aos/configuration.h"
 #include "aos/events/shm_event_loop.h"
+#include "aos/events/simulated_event_loop.h"
 #include "gflags/gflags.h"
 
 namespace aos {
 
+struct PrintOptions {
+  // Format the JSON with the pretty option.
+  bool pretty;
+  // Max vector size to skip expanding.
+  size_t max_vector_size;
+  // Put everything on a separate line instead of keeping small messages
+  // together.
+  bool pretty_max;
+  // Print the timestamps.
+  bool print_timestamps;
+  // Make everything JSON compliant.
+  bool json;
+  // Print the distributed clock.
+  bool distributed_clock;
+  // Print numbers out in hex.
+  bool use_hex;
+};
+
+// Print the flatbuffer out to stdout, both to remove the unnecessary cruft from
+// glog and to allow the user to readily redirect just the logged output
+// independent of any debugging information on stderr.
+void PrintMessage(const std::string_view node_name,
+                  aos::NodeEventLoopFactory *node_factory,
+                  const aos::Channel *channel, const aos::Context &context,
+                  aos::FastStringBuilder *builder, PrintOptions options);
+
+void PrintMessage(const aos::Channel *channel, const aos::Context &context,
+                  aos::FastStringBuilder *builder, PrintOptions options);
+
 // The information needed by the main function of a CLI tool.
 struct CliUtilInfo {
   // If this returns true, main should return immediately with 0.
diff --git a/aos/aos_dump.cc b/aos/aos_dump.cc
index 2fe4955..9f3d01f 100644
--- a/aos/aos_dump.cc
+++ b/aos/aos_dump.cc
@@ -8,8 +8,9 @@
 #include "aos/json_to_flatbuffer.h"
 #include "gflags/gflags.h"
 
-DEFINE_int32(max_vector_size, 100,
+DEFINE_int64(max_vector_size, 100,
              "If positive, vectors longer than this will not be printed");
+DEFINE_bool(json, false, "If true, print fully valid JSON");
 DEFINE_bool(fetch, false,
             "If true, fetch the current message on the channel first");
 DEFINE_bool(pretty, false,
@@ -26,46 +27,7 @@
 DEFINE_int32(timeout, -1,
              "The max time in milliseconds to wait for messages before "
              "exiting.  -1 means forever, 0 means don't wait.");
-
-namespace {
-
-void PrintMessage(const aos::Channel *channel, const aos::Context &context,
-                  aos::FastStringBuilder *builder) {
-  // Print the flatbuffer out to stdout, both to remove the
-  // unnecessary cruft from glog and to allow the user to readily
-  // redirect just the logged output independent of any debugging
-  // information on stderr.
-
-  builder->Reset();
-
-  CHECK(flatbuffers::Verify(*channel->schema(),
-                            *channel->schema()->root_table(),
-                            static_cast<const uint8_t *>(context.data),
-                            static_cast<size_t>(context.size)))
-      << ": Corrupted flatbuffer on " << channel->name()->c_str() << " "
-      << channel->type()->c_str();
-
-  aos::FlatbufferToJson(
-      builder, channel->schema(), static_cast<const uint8_t *>(context.data),
-      {FLAGS_pretty, static_cast<size_t>(FLAGS_max_vector_size),
-       FLAGS_pretty_max});
-
-  if (FLAGS_print_timestamps) {
-    if (context.monotonic_remote_time != context.monotonic_event_time) {
-      std::cout << context.realtime_remote_time << " ("
-                << context.monotonic_remote_time << ") delivered "
-                << context.realtime_event_time << " ("
-                << context.monotonic_event_time << "): " << *builder << '\n';
-    } else {
-      std::cout << context.realtime_event_time << " ("
-                << context.monotonic_event_time << "): " << *builder << '\n';
-    }
-  } else {
-    std::cout << *builder << '\n';
-  }
-}
-
-}  // namespace
+DEFINE_bool(use_hex, false, "Are integers in the messages printed in hex notation.");
 
 int main(int argc, char **argv) {
   gflags::SetUsageMessage(
@@ -93,12 +55,23 @@
 
   aos::monotonic_clock::time_point next_send_time =
       aos::monotonic_clock::min_time;
+
   for (const aos::Channel *channel : cli_info.found_channels) {
     if (FLAGS_fetch) {
       const std::unique_ptr<aos::RawFetcher> fetcher =
           cli_info.event_loop->MakeRawFetcher(channel);
       if (fetcher->Fetch()) {
-        PrintMessage(channel, fetcher->context(), &str_builder);
+        PrintMessage(
+            channel, fetcher->context(), &str_builder,
+            {
+                .pretty = FLAGS_pretty,
+                .max_vector_size = static_cast<size_t>(FLAGS_max_vector_size),
+                .pretty_max = FLAGS_pretty_max,
+                .print_timestamps = FLAGS_print_timestamps,
+                .json = FLAGS_json,
+                .distributed_clock = false,
+                .use_hex = FLAGS_use_hex,
+            });
         ++message_count;
       }
     }
@@ -119,7 +92,17 @@
             if (FLAGS_count > 0 && message_count >= FLAGS_count) {
               return;
             }
-            PrintMessage(channel, context, &str_builder);
+            PrintMessage(channel, context, &str_builder,
+                         {
+                             .pretty = FLAGS_pretty,
+                             .max_vector_size =
+                                 static_cast<size_t>(FLAGS_max_vector_size),
+                             .pretty_max = FLAGS_pretty_max,
+                             .print_timestamps = FLAGS_print_timestamps,
+                             .json = FLAGS_json,
+                             .distributed_clock = false,
+                             .use_hex = FLAGS_use_hex,
+                         });
             ++message_count;
             next_send_time = context.monotonic_event_time +
                              std::chrono::milliseconds(FLAGS_rate_limit);
diff --git a/aos/events/BUILD b/aos/events/BUILD
index 6abc075..0f23d5a 100644
--- a/aos/events/BUILD
+++ b/aos/events/BUILD
@@ -24,10 +24,8 @@
     name = "event_loop_fbs",
     srcs = ["event_loop.fbs"],
     gen_reflections = 1,
-    includes = [
-        "//aos:configuration_fbs_includes",
-    ],
     target_compatible_with = ["@platforms//os:linux"],
+    deps = ["//aos:configuration_fbs"],
 )
 
 cc_static_flatbuffer(
@@ -341,6 +339,7 @@
     deps = [
         ":event_loop_fbs",
         "//aos:configuration",
+        "//aos/util:error_counter",
         "@com_github_google_glog//:glog",
     ],
 )
diff --git a/aos/events/event_loop.cc b/aos/events/event_loop.cc
index 09a3834..257313b 100644
--- a/aos/events/event_loop.cc
+++ b/aos/events/event_loop.cc
@@ -389,19 +389,13 @@
 
   // Pre-fill in the defaults for senders.
   std::vector<flatbuffers::Offset<timing::Sender>> sender_offsets;
-  for (const RawSender *sender : senders_) {
+  for (RawSender *sender : senders_) {
     flatbuffers::Offset<timing::Statistic> size_offset =
         timing::CreateStatistic(fbb);
 
-    std::vector<flatbuffers::Offset<timing::SendErrorCount>>
-        error_count_offsets;
-    for (size_t ii = 0; ii < internal::RawSenderTiming::kNumErrors; ++ii) {
-      error_count_offsets.push_back(timing::CreateSendErrorCount(
-          fbb, timing::EnumValuesSendError()[ii], 0));
-    }
     const flatbuffers::Offset<
         flatbuffers::Vector<flatbuffers::Offset<timing::SendErrorCount>>>
-        error_counts_offset = fbb.CreateVector(error_count_offsets);
+        error_counts_offset = sender->timing_.error_counter.Initialize(&fbb);
 
     timing::Sender::Builder sender_builder(fbb);
 
diff --git a/aos/events/logging/BUILD b/aos/events/logging/BUILD
index 0afdda0..e32da37 100644
--- a/aos/events/logging/BUILD
+++ b/aos/events/logging/BUILD
@@ -310,6 +310,7 @@
     visibility = ["//visibility:public"],
     deps = [
         ":log_reader",
+        "//aos:aos_cli_utils",
         "//aos:configuration",
         "//aos:init",
         "//aos:json_to_flatbuffer",
diff --git a/aos/events/logging/log_cat.cc b/aos/events/logging/log_cat.cc
index 3e1dd1e..1f166e4 100644
--- a/aos/events/logging/log_cat.cc
+++ b/aos/events/logging/log_cat.cc
@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "absl/strings/escaping.h"
+#include "aos/aos_cli_utils.h"
 #include "aos/configuration.h"
 #include "aos/events/logging/log_reader.h"
 #include "aos/events/simulated_event_loop.h"
@@ -33,10 +34,14 @@
 DEFINE_bool(format_raw, true,
             "If true and --raw is specified, print out raw data, but use the "
             "schema to format the data.");
-DEFINE_int32(max_vector_size, 100,
+DEFINE_int64(max_vector_size, 100,
              "If positive, vectors longer than this will not be printed");
 DEFINE_bool(pretty, false,
             "If true, pretty print the messages on multiple lines");
+DEFINE_bool(
+    pretty_max, false,
+    "If true, expand every field to its own line (expands more than -pretty)");
+DEFINE_bool(print_timestamps, true, "If true, timestamps are printed.");
 DEFINE_bool(print, true,
             "If true, actually print the messages.  If false, discard them, "
             "confirming they can be parsed.");
@@ -54,95 +59,11 @@
 DEFINE_double(monotonic_end_time, 0.0,
               "If set, only print messages sent at or before this many seconds "
               "after epoch.");
+DEFINE_bool(use_hex, false, "Are integers in the messages printed in hex notation.");
 
 using aos::monotonic_clock;
 namespace chrono = std::chrono;
 
-void StreamSeconds(std::ostream &stream,
-                   const aos::monotonic_clock::time_point now) {
-  if (now < monotonic_clock::epoch()) {
-    chrono::seconds seconds =
-        chrono::duration_cast<chrono::seconds>(now.time_since_epoch());
-
-    stream << "-" << -seconds.count() << "." << std::setfill('0')
-           << std::setw(9)
-           << chrono::duration_cast<chrono::nanoseconds>(seconds -
-                                                         now.time_since_epoch())
-                  .count();
-  } else {
-    chrono::seconds seconds =
-        chrono::duration_cast<chrono::seconds>(now.time_since_epoch());
-    stream << seconds.count() << "." << std::setfill('0') << std::setw(9)
-           << chrono::duration_cast<chrono::nanoseconds>(
-                  now.time_since_epoch() - seconds)
-                  .count();
-  }
-}
-
-// Print the flatbuffer out to stdout, both to remove the unnecessary cruft from
-// glog and to allow the user to readily redirect just the logged output
-// independent of any debugging information on stderr.
-void PrintMessage(const std::string_view node_name,
-                  aos::NodeEventLoopFactory *node_factory,
-                  const aos::Channel *channel, const aos::Context &context,
-                  aos::FastStringBuilder *builder) {
-  builder->Reset();
-  CHECK(flatbuffers::Verify(*channel->schema(),
-                            *channel->schema()->root_table(),
-                            static_cast<const uint8_t *>(context.data),
-                            static_cast<size_t>(context.size)))
-      << ": Corrupted flatbuffer on " << channel->name()->c_str() << " "
-      << channel->type()->c_str();
-
-  aos::FlatbufferToJson(
-      builder, channel->schema(), static_cast<const uint8_t *>(context.data),
-      {FLAGS_pretty, static_cast<size_t>(FLAGS_max_vector_size)});
-
-  if (FLAGS_json) {
-    std::cout << "{";
-    if (!node_name.empty()) {
-      std::cout << "\"node\": \"" << node_name << "\", ";
-    }
-    std::cout << "\"monotonic_event_time\": ";
-    StreamSeconds(std::cout, context.monotonic_event_time);
-    std::cout << ", \"realtime_event_time\": \"" << context.realtime_event_time
-              << "\", ";
-
-    if (context.monotonic_remote_time != context.monotonic_event_time) {
-      std::cout << "\"monotonic_remote_time\": ";
-      StreamSeconds(std::cout, context.monotonic_remote_time);
-      std::cout << ", \"realtime_remote_time\": \""
-                << context.realtime_remote_time << "\", ";
-    }
-
-    std::cout << "\"channel\": "
-              << aos::configuration::StrippedChannelToString(channel)
-              << ", \"data\": " << *builder << "}" << std::endl;
-  } else {
-    if (FLAGS_distributed_clock) {
-      std::cout << node_factory->ToDistributedClock(
-                       context.monotonic_event_time)
-                << " ";
-    }
-    if (!node_name.empty()) {
-      std::cout << node_name << " ";
-    }
-    if (context.monotonic_remote_time != context.monotonic_event_time) {
-      std::cout << context.realtime_event_time << " ("
-                << context.monotonic_event_time << ") sent "
-                << context.realtime_remote_time << " ("
-                << context.monotonic_remote_time << ") "
-                << channel->name()->c_str() << ' ' << channel->type()->c_str()
-                << ": " << *builder << std::endl;
-    } else {
-      std::cout << context.realtime_event_time << " ("
-                << context.monotonic_event_time << ") "
-                << channel->name()->c_str() << ' ' << channel->type()->c_str()
-                << ": " << *builder << std::endl;
-    }
-  }
-}
-
 // Prints out raw log parts to stdout.
 int PrintRaw(int argc, char **argv) {
   if (argc != 2) {
@@ -332,7 +253,17 @@
             return;
           }
 
-          PrintMessage(node_name_, node_factory_, channel, context, builder_);
+          PrintMessage(
+              node_name_, node_factory_, channel, context, builder_,
+              {
+                  .pretty = FLAGS_pretty,
+                  .max_vector_size = static_cast<size_t>(FLAGS_max_vector_size),
+                  .pretty_max = FLAGS_pretty_max,
+                  .print_timestamps = FLAGS_print_timestamps,
+                  .json = FLAGS_json,
+                  .distributed_clock = FLAGS_distributed_clock,
+                  .use_hex = FLAGS_use_hex,
+              });
           ++(*message_print_counter_);
           if (FLAGS_count > 0 && *message_print_counter_ >= FLAGS_count) {
             factory_->Exit();
diff --git a/aos/events/ping.cc b/aos/events/ping.cc
index a759a8f..9333b61 100644
--- a/aos/events/ping.cc
+++ b/aos/events/ping.cc
@@ -6,7 +6,7 @@
 #include "gflags/gflags.h"
 #include "glog/logging.h"
 
-DEFINE_string(config, "aos/events/pingpong_config.json", "Path to the config.");
+DEFINE_string(config, "pingpong_config.json", "Path to the config.");
 
 int main(int argc, char **argv) {
   aos::InitGoogle(&argc, &argv);
diff --git a/aos/events/pong.cc b/aos/events/pong.cc
index 2f5a0a5..59ddf5d 100644
--- a/aos/events/pong.cc
+++ b/aos/events/pong.cc
@@ -6,7 +6,7 @@
 #include "aos/init.h"
 #include "glog/logging.h"
 
-DEFINE_string(config, "aos/events/pingpong_config.json", "Path to the config.");
+DEFINE_string(config, "pingpong_config.json", "Path to the config.");
 
 int main(int argc, char **argv) {
   aos::InitGoogle(&argc, &argv);
diff --git a/aos/events/timing_statistics.cc b/aos/events/timing_statistics.cc
index fbc49f4..c89bebd 100644
--- a/aos/events/timing_statistics.cc
+++ b/aos/events/timing_statistics.cc
@@ -28,8 +28,10 @@
   sender = new_sender;
   if (!sender) {
     size.set_statistic(nullptr);
+    error_counter.InvalidateBuffer();
   } else {
     size.set_statistic(sender->mutable_size());
+    error_counter.set_mutable_vector(sender->mutable_error_counts());
   }
 }
 
@@ -40,19 +42,14 @@
 
   size.Reset();
   sender->mutate_count(0);
-  for (size_t ii = 0; ii < kNumErrors; ++ii) {
-    sender->mutable_error_counts()->GetMutableObject(ii)->mutate_count(0);
-  }
+  error_counter.ResetCounts();
 }
 
 void RawSenderTiming::IncrementError(timing::SendError error) {
   if (!sender) {
     return;
   }
-  const size_t index = static_cast<size_t>(error);
-  timing::SendErrorCount *counter =
-      sender->mutable_error_counts()->GetMutableObject(index);
-  counter->mutate_count(counter->count() + 1);
+  error_counter.IncrementError(error);
 }
 
 void TimerTiming::set_timing_report(timing::Timer *new_timer) {
diff --git a/aos/events/timing_statistics.h b/aos/events/timing_statistics.h
index e9edff2..3c8c741 100644
--- a/aos/events/timing_statistics.h
+++ b/aos/events/timing_statistics.h
@@ -4,6 +4,7 @@
 #include <cmath>
 
 #include "aos/events/event_loop_generated.h"
+#include "aos/util/error_counter.h"
 
 namespace aos {
 namespace internal {
@@ -79,9 +80,9 @@
 
 // Class to hold timing information for a raw sender.
 struct RawSenderTiming {
-  static constexpr size_t kNumErrors =
-      static_cast<int>(timing::SendError::MAX) -
-      static_cast<int>(timing::SendError::MIN) + 1;
+  typedef util::ErrorCounter<timing::SendError, timing::SendErrorCount>
+      ErrorCounter;
+  static constexpr size_t kNumErrors = ErrorCounter::kNumErrors;
 
   RawSenderTiming(int new_channel_index) : channel_index(new_channel_index) {}
 
@@ -90,8 +91,6 @@
   void IncrementError(timing::SendError error);
   // Sanity check that the enum values are such that we can just use the enum
   // values themselves as array indices without anything weird happening.
-  static_assert(0 == static_cast<int>(timing::SendError::MIN),
-                "Expected error enum values to start at zero.");
   static_assert(
       sizeof(std::invoke_result<decltype(timing::EnumValuesSendError)>::type) /
               sizeof(timing::SendError) ==
@@ -101,6 +100,7 @@
   const int channel_index;
   timing::Sender *sender = nullptr;
   internal::TimingStatistic size;
+  ErrorCounter error_counter;
 };
 
 // Class to hold timing information for timers.
diff --git a/aos/fast_string_builder.h b/aos/fast_string_builder.h
index 9e111ff..aa6b619 100644
--- a/aos/fast_string_builder.h
+++ b/aos/fast_string_builder.h
@@ -37,7 +37,7 @@
 
   // Append integer to result, converted to string representation.
   template <typename T, typename = std::enable_if_t<std::is_integral_v<T>>>
-  void AppendInt(T val);
+  void AppendInt(T val, bool use_hex = false);
 
   void Append(std::string_view);
 
@@ -70,11 +70,18 @@
 };
 
 template <typename T, typename>
-void FastStringBuilder::AppendInt(T val) {
-  std::size_t index = str_.size();
-  Resize(absl::numbers_internal::kFastToBufferSize);
-  char *end = absl::numbers_internal::FastIntToBuffer(val, str_.data() + index);
-  str_.resize(end - str_.data());
+void FastStringBuilder::AppendInt(T val, bool use_hex) {
+  if (use_hex) {
+    // This is not fast like the decimal path, but hex should be used in limited cases.
+    std::stringstream ss;
+    ss << std::hex << val;
+    str_ += ss.str();
+  } else {
+    std::size_t index = str_.size();
+    Resize(absl::numbers_internal::kFastToBufferSize);
+    char *end = absl::numbers_internal::FastIntToBuffer(val, str_.data() + index);
+    str_.resize(end - str_.data());
+  }
 }
 
 }  // namespace aos
diff --git a/aos/flatbuffer_introspection.cc b/aos/flatbuffer_introspection.cc
index b442af8..ddac795 100644
--- a/aos/flatbuffer_introspection.cc
+++ b/aos/flatbuffer_introspection.cc
@@ -11,34 +11,47 @@
 using reflection::BaseType;
 
 void IntToString(int64_t val, reflection::BaseType type,
-                 FastStringBuilder *out) {
+                 FastStringBuilder *out, bool use_hex) {
+  // For 1-byte types in hex mode, we need to cast to 2 bytes to get the desired output and
+  // not unprintable characters.
+  if (use_hex) {
+    if (BaseType::UByte == type) {
+      out->AppendInt(static_cast<uint16_t>(val), true);
+      return;
+    }
+    if (BaseType::Byte == type) {
+      out->AppendInt(static_cast<int16_t>(val), true);
+      return;
+    }
+  }
+
   switch (type) {
     case BaseType::Bool:
       out->AppendBool(static_cast<bool>(val));
       break;
     case BaseType::UByte:
-      out->AppendInt(static_cast<uint8_t>(val));
+      out->AppendInt(static_cast<uint8_t>(val), use_hex);
       break;
     case BaseType::Byte:
-      out->AppendInt(static_cast<int8_t>(val));
+      out->AppendInt(static_cast<int8_t>(val), use_hex);
       break;
     case BaseType::Short:
-      out->AppendInt(static_cast<int16_t>(val));
+      out->AppendInt(static_cast<int16_t>(val), use_hex);
       break;
     case BaseType::UShort:
-      out->AppendInt(static_cast<uint16_t>(val));
+      out->AppendInt(static_cast<uint16_t>(val), use_hex);
       break;
     case BaseType::Int:
-      out->AppendInt(static_cast<int32_t>(val));
+      out->AppendInt(static_cast<int32_t>(val), use_hex);
       break;
     case BaseType::UInt:
-      out->AppendInt(static_cast<uint32_t>(val));
+      out->AppendInt(static_cast<uint32_t>(val), use_hex);
       break;
     case BaseType::Long:
-      out->AppendInt(static_cast<int64_t>(val));
+      out->AppendInt(static_cast<int64_t>(val), use_hex);
       break;
     case BaseType::ULong:
-      out->AppendInt(static_cast<uint64_t>(val));
+      out->AppendInt(static_cast<uint64_t>(val), use_hex);
       break;
     default:
       out->Append("null");
@@ -84,7 +97,7 @@
 void IntOrEnumToString(
     int64_t val, const reflection::Type *type,
     const flatbuffers::Vector<flatbuffers::Offset<reflection::Enum>> *enums,
-    FastStringBuilder *out) {
+    FastStringBuilder *out, bool use_hex = false) {
   // Check if integer is an enum and print string, otherwise fallback to
   // printing as int.
   if (type->index() > -1 && type->index() < (int32_t)enums->size()) {
@@ -104,9 +117,9 @@
   } else {
     if (type->base_type() == BaseType::Vector ||
         type->base_type() == BaseType::Array) {
-      IntToString(val, type->element(), out);
+      IntToString(val, type->element(), out, use_hex);
     } else {
-      IntToString(val, type->base_type(), out);
+      IntToString(val, type->base_type(), out, use_hex);
     }
   }
 }
@@ -150,7 +163,7 @@
     case BaseType::UInt:
     case BaseType::Long:
     case BaseType::ULong:
-      IntOrEnumToString(GetAnyFieldI(*table, *field), type, enums, out);
+      IntOrEnumToString(GetAnyFieldI(*table, *field), type, enums, out, json_options.use_hex);
       break;
     case BaseType::Float:
     case BaseType::Double:
@@ -228,7 +241,7 @@
           if (flatbuffers::IsInteger(elem_type)) {
             IntOrEnumToString(
                 flatbuffers::GetAnyVectorElemI(vector, elem_type, i), type,
-                enums, out);
+                enums, out, json_options.use_hex);
           } else if (flatbuffers::IsFloat(elem_type)) {
             FloatToString(flatbuffers::GetAnyVectorElemF(vector, elem_type, i),
                           elem_type, out);
diff --git a/aos/json_to_flatbuffer.h b/aos/json_to_flatbuffer.h
index 45920a8..d5e1039 100644
--- a/aos/json_to_flatbuffer.h
+++ b/aos/json_to_flatbuffer.h
@@ -45,6 +45,8 @@
   // more extensive version of multi_line that prints every single field on its
   // own line.
   bool max_multi_line = false;
+  // Will integers be printed in hexadecimal form instead of decimal.
+  bool use_hex = false;
 };
 
 // Converts a flatbuffer into a Json string.
diff --git a/aos/starter/subprocess.cc b/aos/starter/subprocess.cc
index 2886c98..558e5c7 100644
--- a/aos/starter/subprocess.cc
+++ b/aos/starter/subprocess.cc
@@ -270,11 +270,18 @@
     stderr_pipes_.write.reset();
   }
 
-  // argv[0] should be the program name
-  args_.insert(args_.begin(), path_);
+  if (run_as_sudo_) {
+    // For sudo we must supply the actual path
+    args_.insert(args_.begin(), path_);
+    args_.insert(args_.begin(), kSudo);
+  } else {
+    // argv[0] should be the program name
+    args_.insert(args_.begin(), name_);
+  }
 
   std::vector<char *> cargs = CArgs();
-  execvp(path_.c_str(), cargs.data());
+  const char* path = run_as_sudo_ ? kSudo : path_.c_str();
+  execvp(path, cargs.data());
 
   // If we got here, something went wrong
   status_pipes_.write->Write(
diff --git a/aos/starter/subprocess.h b/aos/starter/subprocess.h
index bacc574..99ad053 100644
--- a/aos/starter/subprocess.h
+++ b/aos/starter/subprocess.h
@@ -58,6 +58,10 @@
   Application(const aos::Application *application, aos::EventLoop *event_loop,
               std::function<void()> on_change);
 
+  // executable_name is the actual executable path.
+  // When sudo is not used, name is used as argv[0] when exec'ing
+  // executable_name. When sudo is used it's not possible to pass in a
+  // distinct argv[0].
   Application(std::string_view name, std::string_view executable_name,
               aos::EventLoop *event_loop, std::function<void()> on_change);
 
@@ -87,6 +91,7 @@
   void set_args(std::vector<std::string> args);
   void set_capture_stdout(bool capture);
   void set_capture_stderr(bool capture);
+  void set_run_as_sudo(bool value) { run_as_sudo_ = value; }
 
   bool autostart() const { return autostart_; }
 
@@ -106,6 +111,9 @@
 
  private:
   typedef aos::util::ScopedPipe::PipePair PipePair;
+
+  static constexpr const char* const kSudo{"sudo"};
+
   void set_args(
       const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>
           &args);
@@ -140,6 +148,7 @@
   std::string user_name_;
   std::optional<uid_t> user_;
   std::optional<gid_t> group_;
+  bool run_as_sudo_ = false;
 
   bool capture_stdout_ = false;
   PipePair stdout_pipes_;
diff --git a/aos/util/BUILD b/aos/util/BUILD
index 0808062..5a87251 100644
--- a/aos/util/BUILD
+++ b/aos/util/BUILD
@@ -54,6 +54,28 @@
 )
 
 cc_library(
+    name = "error_counter",
+    hdrs = ["error_counter.h"],
+    target_compatible_with = ["@platforms//os:linux"],
+    deps = [
+        "@com_github_google_flatbuffers//:flatbuffers",
+        "@com_github_google_glog//:glog",
+    ],
+)
+
+cc_test(
+    name = "error_counter_test",
+    srcs = ["error_counter_test.cc"],
+    target_compatible_with = ["@platforms//os:linux"],
+    deps = [
+        ":error_counter",
+        "//aos:flatbuffers",
+        "//aos/events:event_loop_fbs",
+        "//aos/testing:googletest",
+    ],
+)
+
+cc_library(
     name = "mcap_logger",
     srcs = ["mcap_logger.cc"],
     hdrs = ["mcap_logger.h"],
diff --git a/aos/util/error_counter.h b/aos/util/error_counter.h
new file mode 100644
index 0000000..cf6cb7f
--- /dev/null
+++ b/aos/util/error_counter.h
@@ -0,0 +1,71 @@
+#ifndef AOS_UTIL_ERROR_COUNTER_H_
+#define AOS_UTIL_ERROR_COUNTER_H_
+#include "flatbuffers/flatbuffers.h"
+#include "glog/logging.h"
+
+namespace aos::util {
+// Class to manage simple error counters for flatbuffer status message.
+// This presumes that you have a flatbuffer enum type Error which has
+// enum values that are continuous and start at zero. These are then
+// counted by a Count flatbuffer table that is of the format:
+// table Count {
+//   error:Error (id: 0);
+//   count:uint (id: 1);
+// }
+// And which is stored as a vector in the resulting status message,
+// where the index within the vector corresponds with the underlying
+// value of the enum.
+template <typename Error, typename Count>
+class ErrorCounter {
+ public:
+  static constexpr size_t kNumErrors =
+      static_cast<int>(Error::MAX) - static_cast<int>(Error::MIN) + 1;
+  static_assert(0 == static_cast<int>(Error::MIN),
+                "Expected Error enum values to start at zero.");
+  // TODO(james): Is there any good way to check that the values are contiguous?
+  // There's no Error::COUNT, and the method I previously used (checking the
+  // size of the return type of EnumValues*()) requires the user to pass that
+  // method as a template argument.
+  ErrorCounter() = default;
+  static flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Count>>>
+  Initialize(flatbuffers::FlatBufferBuilder *fbb) {
+    std::array<flatbuffers::Offset<Count>, kNumErrors> count_offsets;
+    for (size_t ii = 0; ii < kNumErrors; ++ii) {
+      typename Count::Builder builder(*fbb);
+      builder.add_error(static_cast<Error>(ii));
+      builder.add_count(0);
+      count_offsets[ii] = builder.Finish();
+    }
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Count>>>
+        offset = fbb->CreateVector(count_offsets.data(), count_offsets.size());
+    return offset;
+  }
+
+  void set_mutable_vector(
+      flatbuffers::Vector<flatbuffers::Offset<Count>> *vector) {
+    vector_ = vector;
+  }
+
+  void InvalidateBuffer() { vector_ = nullptr; }
+
+  void IncrementError(Error error) {
+    CHECK_NOTNULL(vector_);
+    DCHECK_LT(static_cast<size_t>(error), vector_->size());
+    Count *counter = vector_->GetMutableObject(static_cast<size_t>(error));
+    counter->mutate_count(counter->count() + 1);
+  }
+
+  // Sets all the error counts to zero.
+  void ResetCounts() {
+    CHECK_NOTNULL(vector_);
+    DCHECK_EQ(vector_->size(), kNumErrors) << this << " vector " << vector_;
+    for (size_t ii = 0; ii < kNumErrors; ++ii) {
+      vector_->GetMutableObject(ii)->mutate_count(0);
+    }
+  }
+
+ private:
+  flatbuffers::Vector<flatbuffers::Offset<Count>> *vector_ = nullptr;
+};
+}  // namespace aos::util
+#endif  // AOS_UTIL_ERROR_COUNTER_H_
diff --git a/aos/util/error_counter_test.cc b/aos/util/error_counter_test.cc
new file mode 100644
index 0000000..2166cea
--- /dev/null
+++ b/aos/util/error_counter_test.cc
@@ -0,0 +1,37 @@
+#include "aos/util/error_counter.h"
+
+#include "aos/events/event_loop_generated.h"
+#include "aos/flatbuffers.h"
+#include "gtest/gtest.h"
+
+namespace aos::util::testing {
+// Exercises the basic API for the ErrorCounter class, ensuring that everything
+// works in the normal case.
+TEST(ErrorCounterTest, ErrorCounter) {
+  ErrorCounter<aos::timing::SendError, aos::timing::SendErrorCount> counter;
+  flatbuffers::FlatBufferBuilder fbb;
+  fbb.ForceDefaults(true);
+  const flatbuffers::Offset<
+      flatbuffers::Vector<flatbuffers::Offset<aos::timing::SendErrorCount>>>
+      counts_offset = counter.Initialize(&fbb);
+  aos::timing::Sender::Builder builder(fbb);
+  builder.add_error_counts(counts_offset);
+  fbb.Finish(builder.Finish());
+  aos::FlatbufferDetachedBuffer<aos::timing::Sender> message = fbb.Release();
+  counter.set_mutable_vector(message.mutable_message()->mutable_error_counts());
+  counter.IncrementError(aos::timing::SendError::MESSAGE_SENT_TOO_FAST);
+  counter.IncrementError(aos::timing::SendError::MESSAGE_SENT_TOO_FAST);
+  counter.IncrementError(aos::timing::SendError::INVALID_REDZONE);
+  ASSERT_EQ(2u, message.message().error_counts()->size());
+  EXPECT_EQ(aos::timing::SendError::MESSAGE_SENT_TOO_FAST,
+            message.message().error_counts()->Get(0)->error());
+  EXPECT_EQ(2u, message.message().error_counts()->Get(0)->count());
+  EXPECT_EQ(aos::timing::SendError::INVALID_REDZONE,
+            message.message().error_counts()->Get(1)->error());
+  EXPECT_EQ(1u, message.message().error_counts()->Get(1)->count());
+
+  counter.ResetCounts();
+  EXPECT_EQ(0u, message.message().error_counts()->Get(0)->count());
+  EXPECT_EQ(0u, message.message().error_counts()->Get(1)->count());
+}
+}  // namespace aos::util::testing
diff --git a/third_party/google-glog/bazel/glog.bzl b/third_party/google-glog/bazel/glog.bzl
index 2e7a8d4..b4825d3 100644
--- a/third_party/google-glog/bazel/glog.bzl
+++ b/third_party/google-glog/bazel/glog.bzl
@@ -95,6 +95,8 @@
         "-DHAVE_SYS_SYSCALL_H",
         # For src/logging.cc to create symlinks.
         "-DHAVE_UNISTD_H",
+        "-DHAVE_EXECINFO_H",
+        "-DHAVE_EXECINFO_BACKTRACE",
         "-fvisibility-inlines-hidden",
         "-fvisibility=hidden",
     ]