Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 1 | From 9e407e0be01695e7b927f5820ade87ee9602c248 Mon Sep 17 00:00:00 2001 |
| 2 | From: Alexander Kanavin <alex.kanavin@gmail.com> |
| 3 | Date: Fri, 15 Sep 2017 17:00:14 +0300 |
| 4 | Subject: [PATCH] Use epoll API on Linux |
| 5 | |
| 6 | Also a couple of other modifications due to epoll having |
| 7 | a different approach to how the working set of fds is defined |
| 8 | and used: |
| 9 | 1) open_client() returns an index into the array of clients |
| 10 | 2) close_client() has a protection against being called twice |
| 11 | with the same client (which would mess up the active_clients |
| 12 | counter) |
| 13 | |
| 14 | Upstream-Status: Submitted [Seebs CC'd by email] |
| 15 | Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> |
| 16 | |
| 17 | --- |
| 18 | enums/exit_status.in | 3 + |
| 19 | pseudo_server.c | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++- |
| 20 | 2 files changed, 190 insertions(+), 2 deletions(-) |
| 21 | |
| 22 | diff --git a/enums/exit_status.in b/enums/exit_status.in |
| 23 | index 6be44d3..88f94cd 100644 |
| 24 | --- a/enums/exit_status.in |
| 25 | +++ b/enums/exit_status.in |
| 26 | @@ -18,3 +18,6 @@ listen_fd, "server loop had no valid listen fd" |
| 27 | pseudo_loaded, "server couldn't get out of pseudo environment" |
| 28 | pseudo_prefix, "couldn't get valid pseudo prefix" |
| 29 | pseudo_invocation, "invalid server command arguments" |
| 30 | +epoll_create, "epoll_create() failed" |
| 31 | +epoll_ctl, "epoll_ctl() failed" |
| 32 | + |
| 33 | diff --git a/pseudo_server.c b/pseudo_server.c |
| 34 | index ff16efd..14d34de 100644 |
| 35 | --- a/pseudo_server.c |
| 36 | +++ b/pseudo_server.c |
| 37 | @@ -40,6 +40,12 @@ |
| 38 | #include "pseudo_client.h" |
| 39 | #include "pseudo_db.h" |
| 40 | |
| 41 | +// This has to come after pseudo includes, as that's where PSEUDO_PORT defines are |
| 42 | +#ifdef PSEUDO_PORT_LINUX |
| 43 | +#include <sys/epoll.h> |
| 44 | +#endif |
| 45 | + |
| 46 | + |
| 47 | static int listen_fd = -1; |
| 48 | |
| 49 | typedef struct { |
| 50 | @@ -59,6 +65,7 @@ static int active_clients = 0, highest_client = 0, max_clients = 0; |
| 51 | |
| 52 | #define LOOP_DELAY 2 |
| 53 | #define DEFAULT_PSEUDO_SERVER_TIMEOUT 30 |
| 54 | +#define EPOLL_MAX_EVENTS 10 |
| 55 | int pseudo_server_timeout = DEFAULT_PSEUDO_SERVER_TIMEOUT; |
| 56 | static int die_peacefully = 0; |
| 57 | static int die_forcefully = 0; |
| 58 | @@ -80,6 +87,9 @@ quit_now(int signal) { |
| 59 | static int messages = 0, responses = 0; |
| 60 | static struct timeval message_time = { .tv_sec = 0 }; |
| 61 | |
| 62 | +#ifdef PSEUDO_PORT_LINUX |
| 63 | +static void pseudo_server_loop_epoll(void); |
| 64 | +#endif |
| 65 | static void pseudo_server_loop(void); |
| 66 | |
| 67 | /* helper function to make a directory, just like mkdir -p. |
| 68 | @@ -369,12 +379,16 @@ pseudo_server_start(int daemonize) { |
| 69 | kill(ppid, SIGUSR1); |
| 70 | } |
| 71 | } |
| 72 | +#ifdef PSEUDO_PORT_LINUX |
| 73 | + pseudo_server_loop_epoll(); |
| 74 | +#else |
| 75 | pseudo_server_loop(); |
| 76 | +#endif |
| 77 | return 0; |
| 78 | } |
| 79 | |
| 80 | /* mess with internal tables as needed */ |
| 81 | -static void |
| 82 | +static unsigned int |
| 83 | open_client(int fd) { |
| 84 | pseudo_client_t *new_clients; |
| 85 | int i; |
| 86 | @@ -390,7 +404,7 @@ open_client(int fd) { |
| 87 | ++active_clients; |
| 88 | if (i > highest_client) |
| 89 | highest_client = i; |
| 90 | - return; |
| 91 | + return i; |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | @@ -414,9 +428,11 @@ open_client(int fd) { |
| 96 | |
| 97 | max_clients += 16; |
| 98 | ++active_clients; |
| 99 | + return max_clients - 16; |
| 100 | } else { |
| 101 | pseudo_diag("error allocating new client, fd %d\n", fd); |
| 102 | close(fd); |
| 103 | + return 0; |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | @@ -433,6 +449,10 @@ close_client(int client) { |
| 108 | client, highest_client); |
| 109 | return; |
| 110 | } |
| 111 | + if (clients[client].fd == -1) { |
| 112 | + pseudo_debug(PDBGF_SERVER, "client %d already closed\n", client); |
| 113 | + return; |
| 114 | + } |
| 115 | close(clients[client].fd); |
| 116 | clients[client].fd = -1; |
| 117 | free(clients[client].tag); |
| 118 | @@ -566,6 +586,171 @@ serve_client(int i) { |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | +#ifdef PSEUDO_PORT_LINUX |
| 123 | +static void pseudo_server_loop_epoll(void) |
| 124 | +{ |
| 125 | + struct sockaddr_un client; |
| 126 | + socklen_t len; |
| 127 | + int i; |
| 128 | + int rc; |
| 129 | + int fd; |
| 130 | + int timeout; |
| 131 | + struct epoll_event ev, events[EPOLL_MAX_EVENTS]; |
| 132 | + int loop_timeout = pseudo_server_timeout; |
| 133 | + |
| 134 | + clients = malloc(16 * sizeof(*clients)); |
| 135 | + |
| 136 | + clients[0].fd = listen_fd; |
| 137 | + clients[0].pid = getpid(); |
| 138 | + |
| 139 | + for (i = 1; i < 16; ++i) { |
| 140 | + clients[i].fd = -1; |
| 141 | + clients[i].pid = 0; |
| 142 | + clients[i].tag = NULL; |
| 143 | + clients[i].program = NULL; |
| 144 | + } |
| 145 | + |
| 146 | + active_clients = 1; |
| 147 | + max_clients = 16; |
| 148 | + highest_client = 0; |
| 149 | + |
| 150 | + pseudo_debug(PDBGF_SERVER, "server loop started.\n"); |
| 151 | + if (listen_fd < 0) { |
| 152 | + pseudo_diag("got into loop with no valid listen fd.\n"); |
| 153 | + exit(PSEUDO_EXIT_LISTEN_FD); |
| 154 | + } |
| 155 | + |
| 156 | + timeout = LOOP_DELAY * 1000; |
| 157 | + |
| 158 | + int epollfd = epoll_create1(0); |
| 159 | + if (epollfd == -1) { |
| 160 | + pseudo_diag("epoll_create1() failed.\n"); |
| 161 | + exit(PSEUDO_EXIT_EPOLL_CREATE); |
| 162 | + } |
| 163 | + ev.events = EPOLLIN; |
| 164 | + ev.data.u64 = 0; |
| 165 | + if (epoll_ctl(epollfd, EPOLL_CTL_ADD, clients[0].fd, &ev) == -1) { |
| 166 | + pseudo_diag("epoll_ctl() failed with listening socket.\n"); |
| 167 | + exit(PSEUDO_EXIT_EPOLL_CTL); |
| 168 | + } |
| 169 | + |
| 170 | + pdb_log_msg(SEVERITY_INFO, NULL, NULL, NULL, "server started (pid %d)", getpid()); |
| 171 | + |
| 172 | + for (;;) { |
| 173 | + rc = epoll_wait(epollfd, events, EPOLL_MAX_EVENTS, timeout); |
| 174 | + if (rc == 0 || (rc == -1 && errno == EINTR)) { |
| 175 | + /* If there's no clients, start timing out. If there |
| 176 | + * are active clients, never time out. |
| 177 | + */ |
| 178 | + if (active_clients == 1) { |
| 179 | + loop_timeout -= LOOP_DELAY; |
| 180 | + /* maybe flush database to disk */ |
| 181 | + pdb_maybe_backup(); |
| 182 | + if (loop_timeout <= 0) { |
| 183 | + pseudo_debug(PDBGF_SERVER, "no more clients, got bored.\n"); |
| 184 | + die_peacefully = 1; |
| 185 | + } else { |
| 186 | + /* display this if not exiting */ |
| 187 | + pseudo_debug(PDBGF_SERVER | PDBGF_BENCHMARK, "%d messages handled in %.4f seconds, %d responses\n", |
| 188 | + messages, |
| 189 | + (double) message_time.tv_sec + |
| 190 | + (double) message_time.tv_usec / 1000000.0, |
| 191 | + responses); |
| 192 | + } |
| 193 | + } |
| 194 | + } else if (rc > 0) { |
| 195 | + loop_timeout = pseudo_server_timeout; |
| 196 | + for (i = 0; i < rc; ++i) { |
| 197 | + if (clients[events[i].data.u64].fd == listen_fd) { |
| 198 | + if (!die_forcefully) { |
| 199 | + len = sizeof(client); |
| 200 | + if ((fd = accept(listen_fd, (struct sockaddr *) &client, &len)) != -1) { |
| 201 | + /* Don't allow clients to end up on fd 2, because glibc's |
| 202 | + * malloc debug uses that fd unconditionally. |
| 203 | + */ |
| 204 | + if (fd == 2) { |
| 205 | + int newfd = fcntl(fd, F_DUPFD, 3); |
| 206 | + close(fd); |
| 207 | + fd = newfd; |
| 208 | + } |
| 209 | + pseudo_debug(PDBGF_SERVER, "new client fd %d\n", fd); |
| 210 | + /* A new client implicitly cancels any |
| 211 | + * previous shutdown request, or a |
| 212 | + * shutdown for lack of clients. |
| 213 | + */ |
| 214 | + pseudo_server_timeout = DEFAULT_PSEUDO_SERVER_TIMEOUT; |
| 215 | + die_peacefully = 0; |
| 216 | + |
| 217 | + ev.events = EPOLLIN; |
| 218 | + ev.data.u64 = open_client(fd); |
| 219 | + if (ev.data.u64 != 0 && epoll_ctl(epollfd, EPOLL_CTL_ADD, clients[ev.data.u64].fd, &ev) == -1) { |
| 220 | + pseudo_diag("epoll_ctl() failed with accepted socket.\n"); |
| 221 | + exit(PSEUDO_EXIT_EPOLL_CTL); |
| 222 | + } |
| 223 | + } else if (errno == EMFILE) { |
| 224 | + pseudo_debug(PDBGF_SERVER, "Hit max open files, dropping a client.\n"); |
| 225 | + /* In theory there is a potential race here where if we close a client, |
| 226 | + it may have sent us a fastop message which we don't act upon. |
| 227 | + If we don't close a filehandle we'll loop indefinitely thought. |
| 228 | + Only close one per loop iteration in the interests of caution */ |
| 229 | + for (int j = 1; j <= highest_client; ++j) { |
| 230 | + if (clients[j].fd != -1) { |
| 231 | + close_client(j); |
| 232 | + break; |
| 233 | + } |
| 234 | + } |
| 235 | + } |
| 236 | + } |
| 237 | + } else { |
| 238 | + struct timeval tv1, tv2; |
| 239 | + int rc; |
| 240 | + gettimeofday(&tv1, NULL); |
| 241 | + rc = serve_client(events[i].data.u64); |
| 242 | + gettimeofday(&tv2, NULL); |
| 243 | + ++messages; |
| 244 | + if (rc == 0) |
| 245 | + ++responses; |
| 246 | + message_time.tv_sec += (tv2.tv_sec - tv1.tv_sec); |
| 247 | + message_time.tv_usec += (tv2.tv_usec - tv1.tv_usec); |
| 248 | + if (message_time.tv_usec < 0) { |
| 249 | + message_time.tv_usec += 1000000; |
| 250 | + --message_time.tv_sec; |
| 251 | + } else while (message_time.tv_usec > 1000000) { |
| 252 | + message_time.tv_usec -= 1000000; |
| 253 | + ++message_time.tv_sec; |
| 254 | + } |
| 255 | + } |
| 256 | + if (die_forcefully) |
| 257 | + break; |
| 258 | + } |
| 259 | + pseudo_debug(PDBGF_SERVER, "server loop complete [%d clients left]\n", active_clients); |
| 260 | + } else { |
| 261 | + pseudo_diag("epoll_wait failed: %s\n", strerror(errno)); |
| 262 | + break; |
| 263 | + } |
| 264 | + if (die_peacefully || die_forcefully) { |
| 265 | + pseudo_debug(PDBGF_SERVER, "quitting.\n"); |
| 266 | + pseudo_debug(PDBGF_SERVER | PDBGF_BENCHMARK, "server %d exiting: handled %d messages in %.4f seconds\n", |
| 267 | + getpid(), messages, |
| 268 | + (double) message_time.tv_sec + |
| 269 | + (double) message_time.tv_usec / 1000000.0); |
| 270 | + pdb_log_msg(SEVERITY_INFO, NULL, NULL, NULL, "server %d exiting: handled %d messages in %.4f seconds", |
| 271 | + getpid(), messages, |
| 272 | + (double) message_time.tv_sec + |
| 273 | + (double) message_time.tv_usec / 1000000.0); |
| 274 | + /* and at this point, we'll start refusing connections */ |
| 275 | + close(clients[0].fd); |
| 276 | + /* This is a good place to insert a delay for |
| 277 | + * debugging race conditions during startup. */ |
| 278 | + /* usleep(300000); */ |
| 279 | + exit(0); |
| 280 | + } |
| 281 | + } |
| 282 | + |
| 283 | +} |
| 284 | + |
| 285 | +#endif |
| 286 | + |
| 287 | /* get clients, handle messages, shut down. |
| 288 | * This doesn't actually do any work, it just calls a ton of things which |
| 289 | * do work. |
| 290 | -- |
| 291 | 2.14.1 |
| 292 | |