commit a07b3ce22b9a5403b7056e3bb80431a012d024c6
author Austin Schuh <austin.linux@gmail.com> Sun Oct 10 12:33:21 2021 -0700
committer Austin Schuh <austin.linux@gmail.com> Wed Oct 13 19:56:00 2021 -0700
tree 51ea45affc96958259006ca939b92c24a1b49408
parent d8a0470c445aa65ea2825627f3ebbe53c5b7b612

Slow down restart rate for starterd

1 second delay is too short and results in cascading failures.  Shutdown
isn't actually RT, so it can take down other parts of the system.

Change-Id: Iea364e05ef4547cb2a9755efbf98fb8d488d2f9d
Signed-off-by: Austin Schuh <austin.linux@gmail.com>

aos/starter/starter_cmd.cc

diff --git a/aos/starter/starter_cmd.cc b/aos/starter/starter_cmd.cc
index 91dccd3..95967bb 100644
--- a/aos/starter/starter_cmd.cc
+++ b/aos/starter/starter_cmd.cc
@@ -85,7 +85,7 @@
   const aos::starter::Command command = command_search->second;
   const auto application_name = aos::starter::FindApplication(argv[1], config);
   if (aos::starter::SendCommandBlocking(command, application_name, config,
-                                        chrono::seconds(3))) {
+                                        chrono::seconds(5))) {
     switch (command) {
       case aos::starter::Command::START:
         std::cout << "Successfully started " << application_name << '\n';
@@ -118,7 +118,7 @@
 
         if (aos::starter::SendCommandBlocking(aos::starter::Command::RESTART,
                                               application_name, config,
-                                              chrono::seconds(3))) {
+                                              chrono::seconds(5))) {
           std::cout << "Successfully restarted " << application_name << '\n';
         } else {
           std::cout << "Failed to restart " << application_name << '\n';

aos/starter/starter_test.cc

diff --git a/aos/starter/starter_test.cc b/aos/starter/starter_test.cc
index dea27e9..51f9f02 100644
--- a/aos/starter/starter_test.cc
+++ b/aos/starter/starter_test.cc
@@ -152,7 +152,7 @@
         watcher_loop.Exit();
         FAIL();
       })
-      ->Setup(watcher_loop.monotonic_now() + std::chrono::seconds(7));
+      ->Setup(watcher_loop.monotonic_now() + std::chrono::seconds(11));
 
   int test_stage = 0;
   uint64_t id;
@@ -169,6 +169,7 @@
       case 0: {
         if (app_status->has_state() &&
             app_status->state() == aos::starter::State::RUNNING) {
+          LOG(INFO) << "Ping is running";
           test_stage = 1;
           ASSERT_TRUE(app_status->has_pid());
           ASSERT_TRUE(kill(app_status->pid(), SIGINT) != -1);
@@ -182,6 +183,7 @@
         if (app_status->has_state() &&
             app_status->state() == aos::starter::State::RUNNING &&
             app_status->has_id() && app_status->id() != id) {
+          LOG(INFO) << "Ping restarted";
           watcher_loop.Exit();
           SUCCEED();
         }

aos/starter/starterd_lib.cc

diff --git a/aos/starter/starterd_lib.cc b/aos/starter/starterd_lib.cc
index 1c32be3..526f1c8 100644
--- a/aos/starter/starterd_lib.cc
+++ b/aos/starter/starterd_lib.cc
@@ -156,8 +156,8 @@
 void Application::QueueStart() {
   status_ = aos::starter::State::WAITING;
 
-  LOG(INFO) << "Restarting " << name_ << " in 1 second";
-  restart_timer_->Setup(event_loop_->monotonic_now() + std::chrono::seconds(1));
+  LOG(INFO) << "Restarting " << name_ << " in 3 seconds";
+  restart_timer_->Setup(event_loop_->monotonic_now() + std::chrono::seconds(3));
   start_timer_->Disable();
   stop_timer_->Disable();
 }