blob: 5f964115cc224f60496729d113f107444ae66ddb [file] [log] [blame]
Brian Silvermand169fcd2013-02-27 13:18:47 -08001#include <stdio.h>
2#include <stdlib.h>
3#include <sys/types.h>
4#include <fcntl.h>
5#include <sys/inotify.h>
6#include <sys/stat.h>
7#include <sys/ioctl.h>
8#include <assert.h>
9#include <signal.h>
10#include <stdint.h>
11#include <errno.h>
12#include <string.h>
13#include <sys/wait.h>
14
15#include <map>
16#include <functional>
17#include <deque>
18#include <fstream>
19#include <queue>
20#include <list>
21#include <string>
22#include <vector>
23#include <memory>
24
25#include <event2/event.h>
26
27#include "aos/common/logging/logging.h"
28#include "aos/common/logging/logging_impl.h"
29#include "aos/atom_code/init.h"
30#include "aos/common/unique_malloc_ptr.h"
31#include "aos/common/time.h"
Brian Silverman5cc661b2013-02-27 15:23:36 -080032#include "aos/common/once.h"
Brian Silvermand169fcd2013-02-27 13:18:47 -080033
34// This is the main piece of code that starts all of the rest of the code and
35// restarts it when the binaries are modified.
36//
Brian Silvermanbc4fc2f2013-02-27 19:33:42 -080037// Throughout, the code is not terribly concerned with thread safety because
38// there is only 1 thread. It does some setup and then lets inotify run things
39// when appropriate.
40//
Brian Silverman5cc661b2013-02-27 15:23:36 -080041// NOTE: This program should never exit nicely. It catches all nice attempts to
42// exit, forwards them to all of the children that it has started, waits for
Brian Silvermand169fcd2013-02-27 13:18:47 -080043// them to exit nicely, and then SIGKILLs anybody left (which will always
44// include itself).
45
46using ::std::unique_ptr;
47
48namespace aos {
49namespace starter {
50
Brian Silverman0eec9532013-02-27 20:24:16 -080051// TODO(brians): split out the c++ libevent wrapper stuff into its own file(s)
Brian Silvermand169fcd2013-02-27 13:18:47 -080052class EventBaseDeleter {
53 public:
54 void operator()(event_base *base) {
Brian Silvermand169fcd2013-02-27 13:18:47 -080055 event_base_free(base);
56 }
57};
58typedef unique_ptr<event_base, EventBaseDeleter> EventBaseUniquePtr;
Brian Silverman5cc661b2013-02-27 15:23:36 -080059EventBaseUniquePtr libevent_base;
Brian Silvermand169fcd2013-02-27 13:18:47 -080060
61class EventDeleter {
62 public:
63 void operator()(event *evt) {
Brian Silvermand169fcd2013-02-27 13:18:47 -080064 if (event_del(evt) != 0) {
65 LOG(WARNING, "event_del(%p) failed\n", evt);
66 }
67 }
68};
69typedef unique_ptr<event, EventDeleter> EventUniquePtr;
70
Brian Silverman5cc661b2013-02-27 15:23:36 -080071// Watches a file path for modifications. Once created, keeps watching until
72// destroyed or RemoveWatch() is called.
Brian Silverman0eec9532013-02-27 20:24:16 -080073// TODO(brians): split this out into its own file + tests
Brian Silvermand169fcd2013-02-27 13:18:47 -080074class FileWatch {
75 public:
76 // Will call callback(value) when filename is modified.
77 // If value is NULL, then a pointer to this object will be passed instead.
Brian Silverman5cc661b2013-02-27 15:23:36 -080078 //
79 // Watching for file creations is slightly different. To do that, pass true
80 // for create, the directory where the file will be created for filename, and
81 // the name of the file (without directory name) for check_filename.
Brian Silvermand169fcd2013-02-27 13:18:47 -080082 FileWatch(std::string filename,
83 std::function<void(void *)> callback, void *value,
84 bool create = false, std::string check_filename = "")
85 : filename_(filename), callback_(callback), value_(value),
86 check_filename_(check_filename) {
Brian Silverman5cc661b2013-02-27 15:23:36 -080087 init_once.Get();
88
Brian Silvermand169fcd2013-02-27 13:18:47 -080089 watch_ = inotify_add_watch(notify_fd, filename.c_str(),
90 create ? IN_CREATE : (IN_ATTRIB | IN_MODIFY));
91 if (watch_ == -1) {
Brian Silverman5cc661b2013-02-27 15:23:36 -080092 LOG(FATAL, "inotify_add_watch(%d, %s,"
93 " %s ? IN_CREATE : (IN_ATTRIB | IN_MODIFY)) failed with %d: %s\n",
94 notify_fd, filename.c_str(), create ? "true" : "false",
95 errno, strerror(errno));
Brian Silvermand169fcd2013-02-27 13:18:47 -080096 }
97 watchers[watch_] = this;
98 }
99 // Cleans up everything.
100 ~FileWatch() {
101 if (watch_ != -1) {
102 RemoveWatch();
103 }
104 }
105
106 // After calling this method, this object won't really be doing much of
Brian Silverman5cc661b2013-02-27 15:23:36 -0800107 // anything besides possibly running its callback or something.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800108 void RemoveWatch() {
109 assert(watch_ != -1);
Brian Silverman5cc661b2013-02-27 15:23:36 -0800110
Brian Silvermand169fcd2013-02-27 13:18:47 -0800111 if (inotify_rm_watch(notify_fd, watch_) == -1) {
112 LOG(WARNING, "inotify_rm_watch(%d, %d) failed with %d: %s\n",
113 notify_fd, watch_, errno, strerror(errno));
114 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800115
Brian Silvermand169fcd2013-02-27 13:18:47 -0800116 if (watchers[watch_] != this) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800117 LOG(WARNING, "watcher for %s (%p) didn't find itself in the map\n",
118 filename_.c_str(), this);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800119 } else {
120 watchers.erase(watch_);
121 }
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800122 LOG(DEBUG, "removed watch ID %d\n", watch_);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800123 watch_ = -1;
124 }
125
Brian Silverman5cc661b2013-02-27 15:23:36 -0800126 private:
127 // Performs the static initialization. Called by init_once from the
128 // constructor.
129 static void *Init() {
130 notify_fd = inotify_init1(IN_CLOEXEC);
131 EventUniquePtr notify_event(event_new(libevent_base.get(), notify_fd,
132 EV_READ | EV_PERSIST,
133 FileWatch::INotifyReadable, NULL));
134 event_add(notify_event.release(), NULL);
135 return NULL;
Brian Silvermand169fcd2013-02-27 13:18:47 -0800136 }
137
138 // This gets set up as the callback for EV_READ on the inotify file
Brian Silverman5cc661b2013-02-27 15:23:36 -0800139 // descriptor. It calls FileNotified on the appropriate instance.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800140 static void INotifyReadable(int /*fd*/, short /*events*/, void *) {
141 unsigned int to_read;
Brian Silverman5cc661b2013-02-27 15:23:36 -0800142 // Use FIONREAD to figure out how many bytes there are to read.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800143 if (ioctl(notify_fd, FIONREAD, &to_read) < 0) {
144 LOG(FATAL, "FIONREAD(%d, %p) failed with %d: %s\n",
145 notify_fd, &to_read, errno, strerror(errno));
146 }
147 inotify_event *notifyevt = static_cast<inotify_event *>(malloc(to_read));
148 const char *end = reinterpret_cast<char *>(notifyevt) + to_read;
149 aos::unique_c_ptr<inotify_event> freer(notifyevt);
150
151 ssize_t ret = read(notify_fd, notifyevt, to_read);
152 if (ret < 0) {
153 LOG(FATAL, "read(%d, %p, %u) failed with %d: %s\n",
154 notify_fd, notifyevt, to_read, errno, strerror(errno));
155 }
156 if (static_cast<size_t>(ret) != to_read) {
157 LOG(ERROR, "read(%d, %p, %u) returned %zd instead of %u\n",
158 notify_fd, notifyevt, to_read, ret, to_read);
159 return;
160 }
161
Brian Silverman5cc661b2013-02-27 15:23:36 -0800162 // Keep looping through until we get to the end because inotify does return
163 // multiple events at once.
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800164 while (reinterpret_cast<char *>(notifyevt) < end) {
Brian Silvermand169fcd2013-02-27 13:18:47 -0800165 if (watchers.count(notifyevt->wd) != 1) {
Brian Silvermanfe06fe12013-02-27 18:54:58 -0800166 LOG(WARNING, "couldn't find whose watch ID %d is\n", notifyevt->wd);
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800167 } else {
168 watchers[notifyevt->wd]->FileNotified((notifyevt->len > 0) ?
169 notifyevt->name : NULL);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800170 }
Brian Silvermand169fcd2013-02-27 13:18:47 -0800171
172 notifyevt = reinterpret_cast<inotify_event *>(
173 reinterpret_cast<char *>(notifyevt) +
174 sizeof(*notifyevt) + notifyevt->len);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800175 }
176 }
177
Brian Silverman5cc661b2013-02-27 15:23:36 -0800178 // INotifyReadable calls this method whenever the watch for our file triggers.
179 void FileNotified(const char *filename) {
180 assert(watch_ != -1);
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800181 LOG(DEBUG, "got a notification for %s\n", filename_.c_str());
Brian Silverman5cc661b2013-02-27 15:23:36 -0800182
183 if (!check_filename_.empty()) {
184 if (filename == NULL) {
185 return;
186 }
187 if (std::string(filename) != check_filename_) {
188 return;
189 }
190 }
191
192 callback_((value_ == NULL) ? this : value_);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800193 }
194
Brian Silverman5cc661b2013-02-27 15:23:36 -0800195 // To make sure that Init gets called exactly once.
196 static ::aos::Once<void> init_once;
197
Brian Silvermand169fcd2013-02-27 13:18:47 -0800198 const std::string filename_;
199 const std::function<void(void *)> callback_;
200 void *const value_;
201 std::string check_filename_;
202
203 // The watch descriptor or -1 if we don't have one any more.
204 int watch_;
205
Brian Silvermanbc4fc2f2013-02-27 19:33:42 -0800206 // Map from watch IDs to instances of this class.
207 // <https://patchwork.kernel.org/patch/73192/> ("inotify: do not reuse watch
208 // descriptors") says they won't get reused, but that shouldn't be counted on
209 // because we might have a modified/different version/whatever kernel.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800210 static std::map<int, FileWatch *> watchers;
Brian Silverman5cc661b2013-02-27 15:23:36 -0800211 // The inotify(7) file descriptor.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800212 static int notify_fd;
Brian Silvermanbc4fc2f2013-02-27 19:33:42 -0800213
214 DISALLOW_COPY_AND_ASSIGN(FileWatch);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800215};
Brian Silverman5cc661b2013-02-27 15:23:36 -0800216::aos::Once<void> FileWatch::init_once(FileWatch::Init);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800217std::map<int, FileWatch *> FileWatch::watchers;
218int FileWatch::notify_fd;
219
Brian Silverman5cc661b2013-02-27 15:23:36 -0800220// Runs the given command and returns its first line of output (not including
221// the \n). LOG(FATAL)s if the command has an exit status other than 0 or does
222// not print out an entire line.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800223std::string RunCommand(std::string command) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800224 // popen(3) might fail and not set it.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800225 errno = 0;
Brian Silverman5cc661b2013-02-27 15:23:36 -0800226 FILE *pipe = popen(command.c_str(), "r");
227 if (pipe == NULL) {
Brian Silvermand169fcd2013-02-27 13:18:47 -0800228 LOG(FATAL, "popen(\"%s\", \"r\") failed with %d: %s\n",
229 command.c_str(), errno, strerror(errno));
230 }
231
Brian Silverman5cc661b2013-02-27 15:23:36 -0800232 // result_size is how many bytes result is currently allocated to.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800233 size_t result_size = 128, read = 0;
234 unique_c_ptr<char> result(static_cast<char *>(malloc(result_size)));
235 while (true) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800236 // If we filled up the buffer, then realloc(3) it bigger.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800237 if (read == result_size) {
238 result_size *= 2;
239 void *new_result = realloc(result.get(), result_size);
240 if (new_result == NULL) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800241 LOG(FATAL, "realloc(%p, %zd) failed because of %d: %s\n",
242 result.get(), result_size, errno, strerror(errno));
Brian Silvermand169fcd2013-02-27 13:18:47 -0800243 } else {
244 result.release();
245 result = unique_c_ptr<char>(static_cast<char *>(new_result));
246 }
247 }
248
Brian Silverman5cc661b2013-02-27 15:23:36 -0800249 size_t ret = fread(result.get() + read, 1, result_size - read, pipe);
250 // If the read didn't fill up the whole buffer, check to see if it was
251 // because of an error.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800252 if (ret < result_size - read) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800253 if (ferror(pipe)) {
Brian Silvermand169fcd2013-02-27 13:18:47 -0800254 LOG(FATAL, "couldn't finish reading output of \"%s\"\n",
255 command.c_str());
256 }
257 }
258 read += ret;
259 if (read > 0 && result.get()[read - 1] == '\n') {
260 break;
261 }
262
Brian Silverman5cc661b2013-02-27 15:23:36 -0800263 if (feof(pipe)) {
264 LOG(FATAL, "`%s` failed. didn't print a whole line\n", command.c_str());
Brian Silvermand169fcd2013-02-27 13:18:47 -0800265 }
266 }
267
Brian Silverman5cc661b2013-02-27 15:23:36 -0800268 // Get rid of the first \n and anything after it.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800269 *strchrnul(result.get(), '\n') = '\0';
270
Brian Silverman5cc661b2013-02-27 15:23:36 -0800271 int child_status = pclose(pipe);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800272 if (child_status == -1) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800273 LOG(FATAL, "pclose(%p) failed with %d: %s\n", pipe,
Brian Silvermand169fcd2013-02-27 13:18:47 -0800274 errno, strerror(errno));
275 }
276
277 if (child_status != 0) {
278 LOG(FATAL, "`%s` failed. return %d\n", command.c_str(), child_status);
279 }
280
281 return std::string(result.get());
282}
283
284// Will call callback(arg) after time.
285void Timeout(time::Time time, void (*callback)(int, short, void *), void *arg) {
286 EventUniquePtr timeout(evtimer_new(libevent_base.get(), callback, arg));
287 struct timeval time_timeval = time.ToTimeval();
288 evtimer_add(timeout.release(), &time_timeval);
289}
290
291// Represents a child process. It will take care of restarting itself etc.
292class Child {
293 public:
Brian Silverman5cc661b2013-02-27 15:23:36 -0800294 // command is the (space-separated) command to run and its arguments.
295 Child(const std::string &command) : pid_(-1),
Brian Silvermand169fcd2013-02-27 13:18:47 -0800296 restart_timeout_(
Brian Silvermanfe06fe12013-02-27 18:54:58 -0800297 evtimer_new(libevent_base.get(), StaticDoRestart, this)),
298 stat_at_start_valid_(false) {
Brian Silvermand169fcd2013-02-27 13:18:47 -0800299 const char *start, *end;
300 start = command.c_str();
301 while (true) {
302 end = strchrnul(start, ' ');
303 args_.push_back(std::string(start, end - start));
304 start = end + 1;
305 if (*end == '\0') {
306 break;
307 }
308 }
309
Brian Silverman5cc661b2013-02-27 15:23:36 -0800310 original_binary_ = RunCommand("which " + args_[0]);
311 binary_ = original_binary_ + ".stm";
Brian Silvermand169fcd2013-02-27 13:18:47 -0800312
313 watcher_ = unique_ptr<FileWatch>(
Brian Silverman5cc661b2013-02-27 15:23:36 -0800314 new FileWatch(original_binary_, StaticFileModified, this));
Brian Silvermand169fcd2013-02-27 13:18:47 -0800315
316 Start();
317 }
318
319 pid_t pid() { return pid_; }
320
321 // This gets called whenever the actual process dies and should (probably) be
322 // restarted.
323 void ProcessDied() {
324 pid_ = -1;
325 restarts_.push(time::Time::Now());
326 if (restarts_.size() > kMaxRestartsNumber) {
327 time::Time oldest = restarts_.front();
328 restarts_.pop();
329 if ((time::Time::Now() - oldest) > kMaxRestartsTime) {
330 LOG(WARNING, "process %s getting restarted too often\n", name());
331 Timeout(kResumeWait, StaticStart, this);
332 return;
333 }
334 }
335 Start();
336 }
337
338 // Returns a name for logging purposes.
339 const char *name() {
340 return args_[0].c_str();
341 }
342
343 private:
344 struct CheckDiedStatus {
345 Child *self;
346 pid_t old_pid;
347 };
348
349 // How long to wait for a child to die nicely.
350 static const time::Time kProcessDieTime;
351
352 // How long to wait after the file is modified to restart it.
353 // This is important because some programs like modifying the binaries by
354 // writing them in little bits, which results in attempting to start partial
355 // binaries without this.
356 static const time::Time kRestartWaitTime;
357
Brian Silverman5cc661b2013-02-27 15:23:36 -0800358 // Only kMaxRestartsNumber restarts will be allowed in kMaxRestartsTime.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800359 static const time::Time kMaxRestartsTime;
360 static const size_t kMaxRestartsNumber = 5;
361 // How long to wait if it gets restarted too many times.
362 static const time::Time kResumeWait;
Brian Silverman5cc661b2013-02-27 15:23:36 -0800363
Brian Silvermand169fcd2013-02-27 13:18:47 -0800364 static void StaticFileModified(void *self) {
365 static_cast<Child *>(self)->FileModified();
366 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800367
Brian Silvermand169fcd2013-02-27 13:18:47 -0800368 void FileModified() {
369 struct timeval restart_time_timeval = kRestartWaitTime.ToTimeval();
370 // This will reset the timeout again if it hasn't run yet.
371 evtimer_add(restart_timeout_.get(), &restart_time_timeval);
372 }
373
374 static void StaticDoRestart(int, short, void *self) {
375 static_cast<Child *>(self)->DoRestart();
376 }
377
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800378 // Called after somebody else has finished modifying the file.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800379 void DoRestart() {
Brian Silvermanfe06fe12013-02-27 18:54:58 -0800380 if (stat_at_start_valid_) {
381 struct stat current_stat;
382 if (stat(original_binary_.c_str(), &current_stat) == -1) {
383 LOG(FATAL, "stat(%s, %p) failed with %d: %s\n",
384 original_binary_.c_str(), &current_stat, errno, strerror(errno));
385 }
386 if (current_stat.st_mtime == stat_at_start_.st_mtime) {
387 LOG(DEBUG, "ignoring trigger for %s because mtime didn't change\n",
388 name());
389 return;
390 }
391 }
392
Brian Silvermand169fcd2013-02-27 13:18:47 -0800393 if (pid_ != -1) {
394 LOG(DEBUG, "sending SIGTERM to child %d to restart it\n", pid_);
395 if (kill(pid_, SIGTERM) == -1) {
396 LOG(WARNING, "kill(%d, SIGTERM) failed with %d: %s\n",
397 pid_, errno, strerror(errno));
398 }
399 CheckDiedStatus *status = new CheckDiedStatus();
400 status->self = this;
401 status->old_pid = pid_;
402 Timeout(kProcessDieTime, StaticCheckDied, status);
403 }
404 }
405
406 static void StaticCheckDied(int, short, void *status_in) {
407 CheckDiedStatus *status = static_cast<CheckDiedStatus *>(status_in);
408 status->self->CheckDied(status->old_pid);
409 delete status;
410 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800411
412 // Checks to see if the child using the PID old_pid is still running.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800413 void CheckDied(pid_t old_pid) {
414 if (pid_ == old_pid) {
415 LOG(WARNING, "child %d refused to die\n", old_pid);
416 if (kill(old_pid, SIGKILL) == -1) {
417 LOG(WARNING, "kill(%d, SIGKILL) failed with %d: %s\n",
418 old_pid, errno, strerror(errno));
419 }
420 }
421 }
422
423 static void StaticStart(int, short, void *self) {
424 static_cast<Child *>(self)->Start();
425 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800426
427 // Actually starts the child.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800428 void Start() {
429 if (pid_ != -1) {
430 LOG(WARNING, "calling Start() but already have child %d running\n",
431 pid_);
Brian Silverman5cc661b2013-02-27 15:23:36 -0800432 if (kill(pid_, SIGKILL) == -1) {
433 LOG(WARNING, "kill(%d, SIGKILL) failed with %d: %s\n",
434 pid_, errno, strerror(errno));
435 return;
436 }
Brian Silvermand169fcd2013-02-27 13:18:47 -0800437 pid_ = -1;
438 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800439
440 // Remove the name that we run from (ie from a previous execution) and then
441 // hard link the real filename to it.
442 if (unlink(binary_.c_str()) != 0 && errno != ENOENT) {
443 LOG(FATAL, "removing %s failed because of %d: %s\n",
444 binary_.c_str(), errno, strerror(errno));
445 }
446 if (link(original_binary_.c_str(), binary_.c_str()) != 0) {
447 LOG(FATAL, "link('%s', '%s') failed because of %d: %s\n",
448 original_binary_.c_str(), binary_.c_str(), errno, strerror(errno));
449 }
450
Brian Silvermanfe06fe12013-02-27 18:54:58 -0800451 if (stat(original_binary_.c_str(), &stat_at_start_) == -1) {
452 LOG(FATAL, "stat(%s, %p) failed with %d: %s\n",
453 original_binary_.c_str(), &stat_at_start_, errno, strerror(errno));
454 }
455 stat_at_start_valid_ = true;
456
Brian Silvermand169fcd2013-02-27 13:18:47 -0800457 if ((pid_ = fork()) == 0) {
458 ssize_t args_size = args_.size();
459 const char **argv = new const char *[args_size + 1];
460 for (int i = 0; i < args_size; ++i) {
461 argv[i] = args_[i].c_str();
462 }
463 argv[args_size] = NULL;
464 // The const_cast is safe because no code that might care if it gets
465 // modified can run afterwards.
466 execv(binary_.c_str(), const_cast<char **>(argv));
467 LOG(FATAL, "execv(%s, %p) failed with %d: %s\n",
468 binary_.c_str(), argv, errno, strerror(errno));
469 _exit(EXIT_FAILURE);
470 }
471 if (pid_ == -1) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800472 LOG(FATAL, "forking to run \"%s\" failed with %d: %s\n",
Brian Silvermand169fcd2013-02-27 13:18:47 -0800473 binary_.c_str(), errno, strerror(errno));
474 }
475 }
Brian Silvermanbc4fc2f2013-02-27 19:33:42 -0800476
477 // A history of the times that this process has been restarted.
478 std::queue<time::Time, std::list<time::Time>> restarts_;
479
480 // The currently running child's PID or NULL.
481 pid_t pid_;
482
483 // All of the arguments (including the name of the binary).
484 std::deque<std::string> args_;
485
486 // The name of the real binary that we were told to run.
487 std::string original_binary_;
488 // The name of the file that we're actually running.
489 std::string binary_;
490
491 // Watches original_binary_.
492 unique_ptr<FileWatch> watcher_;
493
494 // An event that restarts after kRestartWaitTime.
495 EventUniquePtr restart_timeout_;
496
497 // Captured from the original file when we most recently started a new child
498 // process. Used to see if it actually changes or not.
499 struct stat stat_at_start_;
500 bool stat_at_start_valid_;
501
502 DISALLOW_COPY_AND_ASSIGN(Child);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800503};
504const time::Time Child::kProcessDieTime = time::Time::InSeconds(0.5);
505const time::Time Child::kMaxRestartsTime = time::Time::InSeconds(2);
506const time::Time Child::kResumeWait = time::Time::InSeconds(1.5);
507const time::Time Child::kRestartWaitTime = time::Time::InSeconds(1.5);
508
509// This is where all of the Child instances except core live.
510std::vector<unique_ptr<Child>> children;
Brian Silverman5cc661b2013-02-27 15:23:36 -0800511// A global place to hold on to which child is core.
512unique_ptr<Child> core;
Brian Silvermand169fcd2013-02-27 13:18:47 -0800513
Brian Silverman5cc661b2013-02-27 15:23:36 -0800514// Kills off the entire process group (including ourself).
515void KillChildren(bool try_nice) {
Brian Silvermand169fcd2013-02-27 13:18:47 -0800516 if (try_nice) {
517 static const int kNiceStopSignal = SIGTERM;
518 static const time::Time kNiceWaitTime = time::Time::InSeconds(1);
519
520 // Make sure that we don't just nicely stop ourself...
521 sigset_t mask;
522 sigemptyset(&mask);
523 sigaddset(&mask, kNiceStopSignal);
524 sigprocmask(SIG_BLOCK, &mask, NULL);
525
Brian Silverman5cc661b2013-02-27 15:23:36 -0800526 kill(-getpid(), kNiceStopSignal);
527
528 fflush(NULL);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800529 time::SleepFor(kNiceWaitTime);
530 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800531
Brian Silvermand169fcd2013-02-27 13:18:47 -0800532 // Send SIGKILL to our whole process group, which will forcibly terminate any
533 // of them that are still running (us for sure, maybe more too).
Brian Silverman5cc661b2013-02-27 15:23:36 -0800534 kill(-getpid(), SIGKILL);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800535}
536
Brian Silverman5cc661b2013-02-27 15:23:36 -0800537void ExitHandler() {
538 KillChildren(true);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800539}
Brian Silverman5cc661b2013-02-27 15:23:36 -0800540
541void KillChildrenSignalHandler(int signum) {
542 // If we get SIGSEGV or some other random signal who knows what's happening
543 // and we should just kill everybody immediately.
544 // This is a list of all of the signals that mean some form of "nicely stop".
545 KillChildren(signum == SIGHUP || signum == SIGINT || signum == SIGQUIT ||
Brian Silverman0eec9532013-02-27 20:24:16 -0800546 signum == SIGABRT || signum == SIGPIPE || signum == SIGTERM ||
547 signum == SIGXCPU);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800548}
549
Brian Silverman5cc661b2013-02-27 15:23:36 -0800550// Returns the currently running child with PID pid or an empty unique_ptr.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800551const unique_ptr<Child> &FindChild(pid_t pid) {
552 for (auto it = children.begin(); it != children.end(); ++it) {
553 if (pid == (*it)->pid()) {
554 return *it;
555 }
556 }
557
558 if (pid == core->pid()) {
559 return core;
560 }
561
Brian Silverman5cc661b2013-02-27 15:23:36 -0800562 static const unique_ptr<Child> kNothing;
Brian Silvermand169fcd2013-02-27 13:18:47 -0800563 return kNothing;
564}
565
Brian Silverman5cc661b2013-02-27 15:23:36 -0800566// Gets set up as a libevent handler for SIGCHLD.
567// Handles calling Child::ProcessDied() on the appropriate one.
568void SigCHLDReceived(int /*fd*/, short /*events*/, void *) {
Brian Silvermand169fcd2013-02-27 13:18:47 -0800569 // In a while loop in case we miss any SIGCHLDs.
570 while (true) {
571 siginfo_t infop;
572 infop.si_pid = 0;
573 if (waitid(P_ALL, 0, &infop, WEXITED | WSTOPPED | WNOHANG) != 0) {
574 LOG(WARNING, "waitid failed with %d: %s", errno, strerror(errno));
Brian Silverman5cc661b2013-02-27 15:23:36 -0800575 continue;
Brian Silvermand169fcd2013-02-27 13:18:47 -0800576 }
Brian Silverman5cc661b2013-02-27 15:23:36 -0800577 // If there are no more child process deaths to process.
Brian Silvermand169fcd2013-02-27 13:18:47 -0800578 if (infop.si_pid == 0) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800579 return;
Brian Silvermand169fcd2013-02-27 13:18:47 -0800580 }
581
582 pid_t pid = infop.si_pid;
583 int status = infop.si_status;
584 const unique_ptr<Child> &child = FindChild(pid);
585 if (child) {
586 switch (infop.si_code) {
587 case CLD_EXITED:
588 LOG(WARNING, "child %d (%s) exited with status %d\n",
589 pid, child->name(), status);
590 break;
591 case CLD_DUMPED:
592 LOG(INFO, "child %d actually dumped core. "
593 "falling through to killed by signal case\n", pid);
594 case CLD_KILLED:
595 // If somebody (possibly us) sent it SIGTERM that means that they just
596 // want it to stop, so it stopping isn't a WARNING.
597 LOG((status == SIGTERM) ? DEBUG : WARNING,
598 "child %d (%s) was killed by signal %d (%s)\n",
599 pid, child->name(), status,
600 strsignal(status));
601 break;
602 case CLD_STOPPED:
603 LOG(WARNING, "child %d (%s) was stopped by signal %d "
604 "(giving it a SIGCONT(%d))\n",
605 pid, child->name(), status, SIGCONT);
606 kill(pid, SIGCONT);
607 continue;
608 default:
609 LOG(WARNING, "something happened to child %d (%s) (killing it)\n",
610 pid, child->name());
611 kill(pid, SIGKILL);
612 continue;
613 }
614 } else {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800615 LOG(WARNING, "couldn't find a Child for pid %d\n", pid);
616 return;
Brian Silvermand169fcd2013-02-27 13:18:47 -0800617 }
618
Brian Silverman5cc661b2013-02-27 15:23:36 -0800619 if (child == core) {
620 LOG(FATAL, "core died\n");
621 }
Brian Silvermand169fcd2013-02-27 13:18:47 -0800622 child->ProcessDied();
623 }
624}
625
Brian Silverman5cc661b2013-02-27 15:23:36 -0800626// This is used for communicating the name of the file to read processes to
627// start from main to Run.
628const char *child_list_file;
629
Brian Silvermand169fcd2013-02-27 13:18:47 -0800630void Main() {
631 logging::Init();
Brian Silverman0eec9532013-02-27 20:24:16 -0800632 // TODO(brians): tell logging that using the root logger from here until we
Brian Silvermand169fcd2013-02-27 13:18:47 -0800633 // bring up shm is ok
634
Brian Silverman5cc661b2013-02-27 15:23:36 -0800635 if (setpgid(0 /*self*/, 0 /*make PGID the same as PID*/) != 0) {
636 LOG(FATAL, "setpgid(0, 0) failed with %d: %s\n", errno, strerror(errno));
637 }
Brian Silvermand169fcd2013-02-27 13:18:47 -0800638
639 // Make sure that we kill all children when we exit.
Brian Silverman5cc661b2013-02-27 15:23:36 -0800640 atexit(ExitHandler);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800641 // Do it on some signals too (ones that we otherwise tend to receive and then
642 // leave all of our children going).
Brian Silverman5cc661b2013-02-27 15:23:36 -0800643 signal(SIGHUP, KillChildrenSignalHandler);
644 signal(SIGINT, KillChildrenSignalHandler);
645 signal(SIGQUIT, KillChildrenSignalHandler);
646 signal(SIGILL, KillChildrenSignalHandler);
647 signal(SIGABRT, KillChildrenSignalHandler);
648 signal(SIGFPE, KillChildrenSignalHandler);
649 signal(SIGSEGV, KillChildrenSignalHandler);
650 signal(SIGPIPE, KillChildrenSignalHandler);
651 signal(SIGTERM, KillChildrenSignalHandler);
652 signal(SIGBUS, KillChildrenSignalHandler);
653 signal(SIGXCPU, KillChildrenSignalHandler);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800654
655 libevent_base = EventBaseUniquePtr(event_base_new());
656
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800657 std::string core_touch_file = "/tmp/starter.";
Brian Silvermand169fcd2013-02-27 13:18:47 -0800658 core_touch_file += std::to_string(static_cast<intmax_t>(getpid()));
659 core_touch_file += ".core_touch_file";
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800660 if (system(("touch '" + core_touch_file + "'").c_str()) != 0) {
661 LOG(FATAL, "running `touch '%s'` failed\n", core_touch_file.c_str());
662 }
663 FileWatch core_touch_file_watch(core_touch_file, Run, NULL);
Brian Silvermand169fcd2013-02-27 13:18:47 -0800664 core = unique_ptr<Child>(
Brian Silvermanb1e4f6c2013-02-27 15:42:02 -0800665 new Child("core " + core_touch_file));
Brian Silvermand169fcd2013-02-27 13:18:47 -0800666
667 FILE *pid_file = fopen("/tmp/starter.pid", "w");
668 if (pid_file == NULL) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800669 LOG(FATAL, "fopen(\"/tmp/starter.pid\", \"w\") failed with %d: %s\n",
Brian Silvermand169fcd2013-02-27 13:18:47 -0800670 errno, strerror(errno));
671 } else {
672 if (fprintf(pid_file, "%d", core->pid()) == -1) {
Brian Silverman5cc661b2013-02-27 15:23:36 -0800673 LOG(WARNING, "fprintf(%p, \"%%d\", %d) failed with %d: %s\n",
674 pid_file, core->pid(), errno, strerror(errno));
Brian Silvermand169fcd2013-02-27 13:18:47 -0800675 }
676 fclose(pid_file);
677 }
678
679 LOG(INFO, "waiting for %s to appear\n", core_touch_file.c_str());
680
681 event_base_dispatch(libevent_base.get());
682 LOG(FATAL, "event_base_dispatch(%p) returned\n", libevent_base.get());
683}
684
Brian Silverman0eec9532013-02-27 20:24:16 -0800685// This is the callback for when core creates the file indicating that it has
686// started.
687void Run(void *watch) {
688 // Make it so it doesn't keep on seeing random changes in /tmp.
689 static_cast<FileWatch *>(watch)->RemoveWatch();
690
691 // It's safe now because core is up.
692 aos::InitNRT();
693
694 std::ifstream list_file(child_list_file);
695
696 while (true) {
697 std::string child_name;
698 getline(list_file, child_name);
699 if ((list_file.rdstate() & std::ios_base::eofbit) != 0) {
700 break;
701 }
702 if (list_file.rdstate() != 0) {
703 LOG(FATAL, "reading input file %s failed\n", child_list_file);
704 }
705 children.push_back(unique_ptr<Child>(new Child(child_name)));
706 }
707
708 EventUniquePtr sigchld(event_new(libevent_base.get(), SIGCHLD,
709 EV_SIGNAL | EV_PERSIST,
710 SigCHLDReceived, NULL));
711 event_add(sigchld.release(), NULL);
712}
713
Brian Silvermand169fcd2013-02-27 13:18:47 -0800714} // namespace starter
715} // namespace aos
716
717int main(int argc, char *argv[]) {
718 if (argc < 2) {
719 fputs("starter: error: need an argument specifying what file to use\n",
720 stderr);
721 exit(EXIT_FAILURE);
722 } else if(argc > 2) {
723 fputs("starter: warning: too many arguments\n", stderr);
724 }
725 aos::starter::child_list_file = argv[1];
726
727 aos::starter::Main();
728}