Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 1 | #include "aos/events/epoll.h" |
| 2 | |
| 3 | #include <fcntl.h> |
| 4 | #include <sys/epoll.h> |
| 5 | #include <sys/timerfd.h> |
| 6 | #include <unistd.h> |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 7 | |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 8 | #include <atomic> |
| 9 | #include <vector> |
| 10 | |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 11 | #include "aos/time/time.h" |
Austin Schuh | f257f3c | 2019-10-27 21:00:43 -0700 | [diff] [blame] | 12 | #include "glog/logging.h" |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 13 | |
| 14 | namespace aos { |
| 15 | namespace internal { |
| 16 | |
| 17 | TimerFd::TimerFd() |
| 18 | : fd_(timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK)) { |
| 19 | PCHECK(fd_ != -1); |
| 20 | Disable(); |
| 21 | } |
| 22 | |
Austin Schuh | f257f3c | 2019-10-27 21:00:43 -0700 | [diff] [blame] | 23 | TimerFd::~TimerFd() { PCHECK(close(fd_) == 0); } |
| 24 | |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 25 | void TimerFd::SetTime(monotonic_clock::time_point start, |
| 26 | monotonic_clock::duration interval) { |
| 27 | struct itimerspec new_value; |
| 28 | new_value.it_interval = ::aos::time::to_timespec(interval); |
| 29 | new_value.it_value = ::aos::time::to_timespec(start); |
| 30 | PCHECK(timerfd_settime(fd_, TFD_TIMER_ABSTIME, &new_value, nullptr) == 0); |
| 31 | } |
| 32 | |
Austin Schuh | de8a8ff | 2019-11-30 15:25:36 -0800 | [diff] [blame] | 33 | uint64_t TimerFd::Read() { |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 34 | uint64_t buf; |
| 35 | ssize_t result = read(fd_, &buf, sizeof(buf)); |
Austin Schuh | de8a8ff | 2019-11-30 15:25:36 -0800 | [diff] [blame] | 36 | if (result == -1) { |
| 37 | if (errno == EAGAIN) { |
| 38 | return 0; |
| 39 | } |
| 40 | } |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 41 | PCHECK(result != -1); |
| 42 | CHECK_EQ(result, static_cast<int>(sizeof(buf))); |
Austin Schuh | de8a8ff | 2019-11-30 15:25:36 -0800 | [diff] [blame] | 43 | |
| 44 | return buf; |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 45 | } |
| 46 | |
| 47 | EPoll::EPoll() : epoll_fd_(epoll_create1(EPOLL_CLOEXEC)) { |
| 48 | PCHECK(epoll_fd_ > 0); |
| 49 | |
| 50 | // Create a pipe for the Quit function. We want to use a pipe to be async |
| 51 | // safe so this can be called from signal handlers. |
| 52 | int pipefd[2]; |
| 53 | PCHECK(pipe2(pipefd, O_CLOEXEC | O_NONBLOCK) == 0); |
| 54 | quit_epoll_fd_ = pipefd[0]; |
| 55 | quit_signal_fd_ = pipefd[1]; |
| 56 | // Read the fd when data is sent and set run_ to false. |
| 57 | OnReadable(quit_epoll_fd_, [this]() { |
| 58 | run_ = false; |
| 59 | char buf[1]; |
| 60 | PCHECK(read(quit_epoll_fd_, &buf[0], 1) == 1); |
| 61 | }); |
| 62 | } |
| 63 | |
| 64 | EPoll::~EPoll() { |
| 65 | // Clean up the quit pipe and epoll fd. |
| 66 | DeleteFd(quit_epoll_fd_); |
| 67 | close(quit_signal_fd_); |
| 68 | close(quit_epoll_fd_); |
Austin Schuh | f257f3c | 2019-10-27 21:00:43 -0700 | [diff] [blame] | 69 | CHECK_EQ(fns_.size(), 0u) |
| 70 | << ": Not all file descriptors were unregistered before shutting down."; |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 71 | close(epoll_fd_); |
| 72 | } |
| 73 | |
Austin Schuh | 219b778 | 2020-12-21 12:01:40 -0800 | [diff] [blame] | 74 | void EPoll::BeforeWait(std::function<void()> function) { |
| 75 | before_epoll_wait_functions_.emplace_back(std::move(function)); |
| 76 | } |
| 77 | |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 78 | void EPoll::Run() { |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 79 | run_ = true; |
James Kuszmaul | c8e657e | 2020-12-14 23:49:45 -0800 | [diff] [blame] | 80 | // As long as run_ is true or we still have events to process, keep polling. |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 81 | while (true) { |
James Kuszmaul | c8e657e | 2020-12-14 23:49:45 -0800 | [diff] [blame] | 82 | // If we ran out of events and Quit() was called, quit |
| 83 | if (!Poll(run_)) { |
| 84 | if (!run_) { |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 85 | return; |
| 86 | } |
| 87 | } |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 88 | } |
| 89 | } |
| 90 | |
James Kuszmaul | c8e657e | 2020-12-14 23:49:45 -0800 | [diff] [blame] | 91 | bool EPoll::Poll(bool block) { |
Austin Schuh | 219b778 | 2020-12-21 12:01:40 -0800 | [diff] [blame] | 92 | for (const std::function<void()> &function : before_epoll_wait_functions_) { |
| 93 | function(); |
| 94 | } |
James Kuszmaul | c8e657e | 2020-12-14 23:49:45 -0800 | [diff] [blame] | 95 | // Pull a single event out. Infinite timeout if we are supposed to be |
| 96 | // running, and 0 length timeout otherwise. This lets us flush the event |
| 97 | // queue before quitting. |
| 98 | struct epoll_event event; |
| 99 | int num_events = epoll_wait(epoll_fd_, &event, 1, block ? -1 : 0); |
| 100 | // Retry on EINTR and nothing else. |
| 101 | if (num_events == -1) { |
| 102 | if (errno == EINTR) { |
| 103 | return false; |
| 104 | } |
| 105 | PCHECK(num_events != -1); |
| 106 | } |
| 107 | |
| 108 | if (num_events == 0) { |
| 109 | return false; |
| 110 | } |
| 111 | |
| 112 | EventData *const event_data = static_cast<struct EventData *>(event.data.ptr); |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 113 | event_data->DoCallbacks(event.events); |
James Kuszmaul | c8e657e | 2020-12-14 23:49:45 -0800 | [diff] [blame] | 114 | return true; |
| 115 | } |
| 116 | |
Austin Schuh | f74daa6 | 2021-07-21 15:29:17 -0700 | [diff] [blame] | 117 | void EPoll::Quit() { |
| 118 | // Shortcut to break us out of infinite loops. We might write more than once |
| 119 | // to the pipe, but we'll stop once the first is read on the other end. |
| 120 | if (!run_) { |
| 121 | return; |
| 122 | } |
| 123 | PCHECK(write(quit_signal_fd_, "q", 1) == 1); |
| 124 | } |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 125 | |
| 126 | void EPoll::OnReadable(int fd, ::std::function<void()> function) { |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 127 | EventData *event_data = GetEventData(fd); |
| 128 | if (event_data == nullptr) { |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 129 | fns_.emplace_back(std::make_unique<InOutEventData>(fd)); |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 130 | event_data = fns_.back().get(); |
| 131 | } else { |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 132 | CHECK(!static_cast<InOutEventData *>(event_data)->in_fn) |
| 133 | << ": Duplicate in functions for " << fd; |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 134 | } |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 135 | static_cast<InOutEventData *>(event_data)->in_fn = ::std::move(function); |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 136 | DoEpollCtl(event_data, event_data->events | kInEvents); |
| 137 | } |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 138 | |
Austin Schuh | 603419f | 2021-04-24 18:15:20 -0700 | [diff] [blame] | 139 | void EPoll::OnError(int fd, ::std::function<void()> function) { |
| 140 | EventData *event_data = GetEventData(fd); |
| 141 | if (event_data == nullptr) { |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 142 | fns_.emplace_back(std::make_unique<InOutEventData>(fd)); |
Austin Schuh | 603419f | 2021-04-24 18:15:20 -0700 | [diff] [blame] | 143 | event_data = fns_.back().get(); |
| 144 | } else { |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 145 | CHECK(!static_cast<InOutEventData *>(event_data)->err_fn) |
| 146 | << ": Duplicate error functions for " << fd; |
Austin Schuh | 603419f | 2021-04-24 18:15:20 -0700 | [diff] [blame] | 147 | } |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 148 | static_cast<InOutEventData *>(event_data)->err_fn = ::std::move(function); |
Austin Schuh | 603419f | 2021-04-24 18:15:20 -0700 | [diff] [blame] | 149 | DoEpollCtl(event_data, event_data->events | kErrorEvents); |
| 150 | } |
| 151 | |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 152 | void EPoll::OnWriteable(int fd, ::std::function<void()> function) { |
| 153 | EventData *event_data = GetEventData(fd); |
| 154 | if (event_data == nullptr) { |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 155 | fns_.emplace_back(std::make_unique<InOutEventData>(fd)); |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 156 | event_data = fns_.back().get(); |
| 157 | } else { |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 158 | CHECK(!static_cast<InOutEventData *>(event_data)->out_fn) |
| 159 | << ": Duplicate out functions for " << fd; |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 160 | } |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 161 | static_cast<InOutEventData *>(event_data)->out_fn = ::std::move(function); |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 162 | DoEpollCtl(event_data, event_data->events | kOutEvents); |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 163 | } |
| 164 | |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 165 | void EPoll::OnEvents(int fd, ::std::function<void(uint32_t)> function) { |
| 166 | if (GetEventData(fd) != nullptr) { |
| 167 | LOG(FATAL) << "May not replace OnEvents handlers"; |
| 168 | } |
| 169 | fns_.emplace_back(std::make_unique<SingleEventData>(fd)); |
| 170 | static_cast<SingleEventData *>(fns_.back().get())->fn = std::move(function); |
| 171 | } |
| 172 | |
Austin Schuh | 219b778 | 2020-12-21 12:01:40 -0800 | [diff] [blame] | 173 | void EPoll::ForgetClosedFd(int fd) { |
| 174 | auto element = fns_.begin(); |
| 175 | while (fns_.end() != element) { |
| 176 | if (element->get()->fd == fd) { |
| 177 | fns_.erase(element); |
| 178 | return; |
| 179 | } |
| 180 | ++element; |
| 181 | } |
| 182 | LOG(FATAL) << "fd " << fd << " not found"; |
| 183 | } |
| 184 | |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 185 | void EPoll::SetEvents(int fd, uint32_t events) { |
| 186 | DoEpollCtl(CHECK_NOTNULL(GetEventData(fd)), events); |
| 187 | } |
| 188 | |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 189 | // Removes fd from the event loop. |
| 190 | void EPoll::DeleteFd(int fd) { |
| 191 | auto element = fns_.begin(); |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 192 | while (fns_.end() != element) { |
| 193 | if (element->get()->fd == fd) { |
| 194 | fns_.erase(element); |
Austin Schuh | 219b778 | 2020-12-21 12:01:40 -0800 | [diff] [blame] | 195 | PCHECK(epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, nullptr) == 0) |
| 196 | << "Failed to delete fd " << fd; |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 197 | return; |
| 198 | } |
| 199 | ++element; |
| 200 | } |
Austin Schuh | f257f3c | 2019-10-27 21:00:43 -0700 | [diff] [blame] | 201 | LOG(FATAL) << "fd " << fd << " not found"; |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 202 | } |
| 203 | |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 204 | void EPoll::InOutEventData::DoCallbacks(uint32_t events) { |
| 205 | if (events & kInEvents) { |
| 206 | CHECK(in_fn) << ": No handler registered for input events on " << fd; |
| 207 | in_fn(); |
| 208 | } |
| 209 | if (events & kOutEvents) { |
| 210 | CHECK(out_fn) << ": No handler registered for output events on " << fd; |
| 211 | out_fn(); |
| 212 | } |
| 213 | if (events & kErrorEvents) { |
| 214 | CHECK(err_fn) << ": No handler registered for error events on " << fd; |
| 215 | err_fn(); |
| 216 | } |
| 217 | } |
| 218 | |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 219 | void EPoll::EnableEvents(int fd, uint32_t events) { |
| 220 | EventData *const event_data = CHECK_NOTNULL(GetEventData(fd)); |
| 221 | DoEpollCtl(event_data, event_data->events | events); |
| 222 | } |
| 223 | |
| 224 | void EPoll::DisableEvents(int fd, uint32_t events) { |
| 225 | EventData *const event_data = CHECK_NOTNULL(GetEventData(fd)); |
| 226 | DoEpollCtl(event_data, event_data->events & ~events); |
| 227 | } |
| 228 | |
| 229 | EPoll::EventData *EPoll::GetEventData(int fd) { |
| 230 | const auto iterator = std::find_if( |
| 231 | fns_.begin(), fns_.end(), |
| 232 | [fd](const std::unique_ptr<EventData> &data) { return data->fd == fd; }); |
| 233 | if (iterator == fns_.end()) { |
| 234 | return nullptr; |
| 235 | } |
| 236 | return iterator->get(); |
| 237 | } |
| 238 | |
| 239 | void EPoll::DoEpollCtl(EventData *event_data, const uint32_t new_events) { |
| 240 | const uint32_t old_events = event_data->events; |
Austin Schuh | 4c83968 | 2021-07-21 15:30:44 -0700 | [diff] [blame] | 241 | if (old_events == new_events) { |
| 242 | // Shortcut without calling into the kernel. This happens often with |
| 243 | // external event loop integrations that are emulating poll, so make it |
| 244 | // fast. |
| 245 | return; |
| 246 | } |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 247 | event_data->events = new_events; |
| 248 | if (new_events == 0) { |
Brian Silverman | 441591b | 2020-01-31 17:44:32 -0800 | [diff] [blame] | 249 | // It was added, but should now be removed. |
| 250 | PCHECK(epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, event_data->fd, nullptr) == 0); |
| 251 | return; |
| 252 | } |
| 253 | |
| 254 | int operation = EPOLL_CTL_MOD; |
| 255 | if (old_events == 0) { |
| 256 | // If it wasn't added before, then this is the first time it's being added. |
| 257 | operation = EPOLL_CTL_ADD; |
| 258 | } |
| 259 | struct epoll_event event; |
| 260 | event.events = event_data->events; |
| 261 | event.data.ptr = event_data; |
| 262 | PCHECK(epoll_ctl(epoll_fd_, operation, event_data->fd, &event) == 0) |
| 263 | << ": Failed to " << operation << " epoll fd: " << event_data->fd; |
| 264 | } |
| 265 | |
Austin Schuh | 6b6dfa5 | 2019-06-12 20:16:20 -0700 | [diff] [blame] | 266 | } // namespace internal |
| 267 | } // namespace aos |