Add AOS docs covering ShmEventLoop & multi-node constructs

Add a few thousand words covering the ShmEventLoop and the basics of how
multi-node systems work.

Still a fair amount to be written on multi-node systems overall.

This also updates/adds to some of the example code we have:

* Add options that allow for experimenting with higher rates & fetchers
  in ping/pong (since I talked about performance benefits of watchers vs
  fetchers in the docs).
* Update the starter_demo script to no longer call `aos_starter`
  `starter_cmd` and to provide the `aos_timing_report_streamer`.
* Add sample code for using the `ServerStatistics` clock offsets.

This change is primarily meant to put factually accurate and clear
information into the docs, and less so about getting perfectly styled
prose.

Change-Id: Ieff45b0ef45e3390e0f98630a65651028206a9f0
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/aos/events/ping_lib.cc b/aos/events/ping_lib.cc
index a7a8dd4..56dde2f 100644
--- a/aos/events/ping_lib.cc
+++ b/aos/events/ping_lib.cc
@@ -7,7 +7,7 @@
 #include "aos/events/pong_generated.h"
 #include "aos/json_to_flatbuffer.h"
 
-DEFINE_int32(sleep_ms, 10, "Time to sleep between pings");
+DEFINE_int32(sleep_us, 10000, "Time to sleep between pings");
 
 namespace aos {
 
@@ -26,7 +26,7 @@
 
   event_loop_->OnRun([this]() {
     timer_handle_->Schedule(event_loop_->monotonic_now(),
-                            chrono::milliseconds(FLAGS_sleep_ms));
+                            chrono::microseconds(FLAGS_sleep_us));
   });
 
   event_loop_->SetRuntimeRealtimePriority(5);
@@ -35,7 +35,7 @@
 void Ping::SendPing() {
   if (last_pong_value_ != count_ && (!quiet_ || VLOG_IS_ON(1))) {
     LOG(WARNING) << "Did not receive response to " << count_ << " within "
-                 << FLAGS_sleep_ms << "ms.";
+                 << FLAGS_sleep_us << "us.";
   }
   ++count_;
   aos::Sender<examples::Ping>::Builder builder = sender_.MakeBuilder();
diff --git a/aos/events/pong_lib.cc b/aos/events/pong_lib.cc
index 93a551e..e8bf171 100644
--- a/aos/events/pong_lib.cc
+++ b/aos/events/pong_lib.cc
@@ -6,27 +6,45 @@
 #include "aos/events/ping_generated.h"
 #include "aos/events/pong_generated.h"
 
+DEFINE_bool(fetch, false, "Poll & fetch messages instead of using a watcher.");
+DEFINE_uint32(fetch_period_ms, 10, "Frequency at which to fetch.");
+
 namespace aos {
 
 Pong::Pong(EventLoop *event_loop)
     : event_loop_(event_loop),
+      fetcher_(event_loop_->MakeFetcher<examples::Ping>("/test")),
       sender_(event_loop_->MakeSender<examples::Pong>("/test")) {
-  event_loop_->MakeWatcher("/test", [this](const examples::Ping &ping) {
-    if (last_value_ == ping.value() && (!quiet_ || VLOG_IS_ON(1))) {
-      LOG(WARNING) << "Duplicate ping value at " << last_value_
-                   << " time difference " << ping.send_time() - last_send_time_;
-    }
-    last_value_ = ping.value();
-    last_send_time_ = ping.send_time();
-    aos::Sender<examples::Pong>::Builder builder = sender_.MakeBuilder();
-    examples::Pong::Builder pong_builder =
-        builder.MakeBuilder<examples::Pong>();
-    pong_builder.add_value(ping.value());
-    pong_builder.add_initial_send_time(ping.send_time());
-    builder.CheckOk(builder.Send(pong_builder.Finish()));
-  });
+  if (FLAGS_fetch) {
+    event_loop_
+        ->AddPhasedLoop(
+            [this](int) {
+              while (fetcher_.FetchNext()) {
+                HandlePing(*fetcher_.get());
+              }
+            },
+            std::chrono::milliseconds(FLAGS_fetch_period_ms))
+        ->set_name("pong");
+  } else {
+    event_loop_->MakeWatcher(
+        "/test", [this](const examples::Ping &ping) { HandlePing(ping); });
+  }
 
   event_loop_->SetRuntimeRealtimePriority(5);
 }
 
+void Pong::HandlePing(const examples::Ping &ping) {
+  if (last_value_ == ping.value() && (!quiet_ || VLOG_IS_ON(1))) {
+    LOG(WARNING) << "Duplicate ping value at " << last_value_
+                 << " time difference " << ping.send_time() - last_send_time_;
+  }
+  last_value_ = ping.value();
+  last_send_time_ = ping.send_time();
+  aos::Sender<examples::Pong>::Builder builder = sender_.MakeBuilder();
+  examples::Pong::Builder pong_builder = builder.MakeBuilder<examples::Pong>();
+  pong_builder.add_value(ping.value());
+  pong_builder.add_initial_send_time(ping.send_time());
+  builder.CheckOk(builder.Send(pong_builder.Finish()));
+}
+
 }  // namespace aos
diff --git a/aos/events/pong_lib.h b/aos/events/pong_lib.h
index 8b6641a..a12dad0 100644
--- a/aos/events/pong_lib.h
+++ b/aos/events/pong_lib.h
@@ -15,7 +15,9 @@
   void set_quiet(bool quiet) { quiet_ = quiet; }
 
  private:
+  void HandlePing(const examples::Ping &ping);
   EventLoop *event_loop_;
+  aos::Fetcher<examples::Ping> fetcher_;
   aos::Sender<examples::Pong> sender_;
   int32_t last_value_ = 0;
   int32_t last_send_time_ = 0;
diff --git a/aos/starter/BUILD b/aos/starter/BUILD
index 7ef3777..732cfe3 100644
--- a/aos/starter/BUILD
+++ b/aos/starter/BUILD
@@ -201,11 +201,13 @@
         "$(rootpath :aos_starter)",
         "$(rootpath //aos:aos_dump)",
         "$(rootpath //aos/events/logging:logger_main)",
+        "$(rootpath //aos/events:aos_timing_report_streamer)",
     ],
     data = [
         ":aos_starter",
         ":starterd",
         "//aos:aos_dump",
+        "//aos/events:aos_timing_report_streamer",
         "//aos/events:ping",
         "//aos/events:pingpong_config",
         "//aos/events:pong",
diff --git a/aos/starter/starter_demo.py b/aos/starter/starter_demo.py
index 5a50890..89d06a0 100755
--- a/aos/starter/starter_demo.py
+++ b/aos/starter/starter_demo.py
@@ -7,9 +7,9 @@
 
 DESCRIPTION = """
 This script provides a convenient way to experiment with the starter
-and starter_cmd in particular. To run this, run:
+and aos_starter in particular. To run this, run:
 $ bazel run  //aos/starter:starter_demo
-This will then print out instructions for running starter_cmd.
+This will then print out instructions for running aos_starter.
 
 If running via bazel, you should not need to specify the positional
 arguments.
@@ -21,11 +21,7 @@
         formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument("starterd", help="Location of starterd")
     parser.add_argument("configs", help="Location of the config files")
-    parser.add_argument("ping", help="Location of ping")
-    parser.add_argument("pong", help="Location of pong")
-    parser.add_argument("starter_cmd", help="Location of starter_cmd")
-    parser.add_argument("aos_dump", help="Location of aos_dump")
-    parser.add_argument("logger_main", help="Location of logger_main")
+    parser.add_argument("binaries", nargs='+', help="Binaries to provide")
     args = parser.parse_args()
 
     # Copy all the interesting files into a temporary directory and run
@@ -37,14 +33,11 @@
     # can take a new --shm_base to allow cleaner running on shared systems.
     with tempfile.TemporaryDirectory() as tmpdir:
         shutil.copy(args.starterd, tmpdir + "/starterd")
-        shutil.copy(args.ping, tmpdir + "/ping")
-        shutil.copy(args.pong, tmpdir + "/pong")
-        shutil.copy(args.starter_cmd, tmpdir + "/starter_cmd")
-        shutil.copy(args.aos_dump, tmpdir + "/aos_dump")
-        shutil.copy(args.logger_main, tmpdir + "/logger_main")
+        for binary in args.binaries:
+            shutil.copy(binary, tmpdir + "/" + os.path.basename(binary))
         print(f"Running starter from {tmpdir}")
 
-        print(f"\n\nTo run starter_cmd, do:\ncd {tmpdir}\n./starter_cmd\n\n")
+        print(f"\n\nTo run aos_starter, do:\ncd {tmpdir}\n./aos_starter\n\n")
 
         for config in args.configs.split(' '):
             basename = os.path.basename(config)