signals: implement safe async signal handler framework

Based on http://evbergen.home.xs4all.nl/unix-signals.html.

Instead of his design, though, we write single bytes, and create a pipe per
signal we have handlers registered for.

Signed-off-by: Sage Weil <sage@newdream.net>

3 files changed, 307 insertions, 0 deletions

diff --git a/src/global/signal_handler.cc b/src/global/signal_handler.cc
index 950315f9cfe..f83a4f3afb8 100644
--- a/src/global/signal_handler.cc
+++ b/src/global/signal_handler.cc
@@ -123,3 +123,231 @@ void install_standard_sighandlers(void)
   install_sighandler(SIGTERM, handle_shutdown_signal, SA_RESETHAND | SA_NODEFER);
   install_sighandler(SIGINT, handle_shutdown_signal, SA_RESETHAND | SA_NODEFER);
 }
+
+
+
+/// --- safe handler ---
+
+#include "common/Thread.h"
+#include <errno.h>
+
+/**
+ * safe async signal handler / dispatcher
+ *
+ * This is an async unix signal handler based on the design from
+ *
+ *  http://evbergen.home.xs4all.nl/unix-signals.html
+ *
+ * Features:
+ *   - no unsafe work is done in the signal handler itself
+ *   - callbacks are called from a regular thread
+ *   - signals are not lost, unless multiple instances of the same signal
+ *     are sent twice in quick succession.
+ */
+struct SignalHandler : public Thread {
+  /// to kick the thread, for shutdown, new handlers, etc.
+  int pipefd[2];  // write to [1], read from [0]
+
+  /// to signal shutdown
+  bool stop;
+
+  /// for an individual signal
+  struct safe_handler {
+    int pipefd[2];  // write to [1], read from [0]
+    signal_handler_t handler;
+  };
+
+  /// all handlers
+  safe_handler *handlers[32];
+
+  /// to protect the handlers array
+  Mutex lock;
+
+  SignalHandler()
+    : stop(false), lock("SignalHandler::lock")
+  {
+    for (unsigned i = 0; i < 32; i++)
+      handlers[i] = NULL;
+
+    // create signal pipe
+    int r = pipe(pipefd);
+    assert(r == 0);
+    r = fcntl(pipefd[0], F_SETFL, O_NONBLOCK);
+    assert(r == 0);
+
+    // create thread
+    create();
+  }
+
+  ~SignalHandler() {
+    shutdown();
+  }
+
+  void signal_thread() {
+    write(pipefd[1], "\0", 1);
+  }
+
+  void shutdown() {
+    stop = true;
+    signal_thread();
+    join();
+  }
+
+  // thread entry point
+  void *entry() {
+    while (!stop) {
+      // create fd set
+      fd_set rfds;
+      FD_ZERO(&rfds);
+      FD_SET(pipefd[0], &rfds);
+      int max_fd = pipefd[0];
+
+      lock.Lock();
+      for (unsigned i=0; i<32; i++) {
+	if (handlers[i]) {
+	  int fd = handlers[i]->pipefd[0];
+	  FD_SET(fd, &rfds);
+	  if (fd > max_fd)
+	    max_fd = fd;
+	}
+      }
+      lock.Unlock();
+
+      // wait for data on any of those pipes
+      int r = select(max_fd + 1, &rfds, NULL, NULL, NULL);
+      if (stop)
+	break;
+      if (r > 0) {
+	char v;
+
+	// consume byte from signal socket, if any.
+	r = read(pipefd[0], &v, 1);
+
+	lock.Lock();
+	for (unsigned signum=0; signum<32; signum++) {
+	  if (handlers[signum]) {
+	    r = read(handlers[signum]->pipefd[0], &v, 1);
+	    if (r == 1) {
+	      handlers[signum]->handler(signum);
+	    }
+	  }
+	}
+	lock.Unlock();
+      } else {
+	//cout << "no data, got r=" << r << " errno=" << errno << std::endl;
+      }
+    }
+    return NULL;
+  }
+
+  void queue_signal(int signum) {
+    // If this signal handler is registered, the callback must be
+    // defined.  We can do this without the lock because we will never
+    // have the signal handler defined without the handlers entry also
+    // being filled in.
+    assert(handlers[signum]);
+    write(handlers[signum]->pipefd[1], " ", 1);
+  }
+
+  void register_handler(int signum, signal_handler_t handler, bool oneshot);
+  void unregister_handler(int signum, signal_handler_t handler);
+};
+
+static SignalHandler *g_signal_handler = NULL;
+
+static void handler_hook(int signum)
+{
+  g_signal_handler->queue_signal(signum);
+}
+
+void SignalHandler::register_handler(int signum, signal_handler_t handler, bool oneshot)
+{
+  int r;
+
+  assert(signum >= 0 && signum < 32);
+
+  safe_handler *h = new safe_handler;
+
+  r = pipe(h->pipefd);
+  assert(r == 0);
+  r = fcntl(h->pipefd[0], F_SETFL, O_NONBLOCK);
+  assert(r == 0);
+
+  h->handler = handler;
+  lock.Lock();
+  handlers[signum] = h;
+  lock.Unlock();
+
+  // signal thread so that it sees our new handler
+  signal_thread();
+  
+  // install our handler
+  struct sigaction oldact;
+  struct sigaction act;
+  memset(&act, 0, sizeof(act));
+
+  act.sa_handler = handler_hook;
+  sigfillset(&act.sa_mask);  // mask all signals in the handler
+  act.sa_flags = oneshot ? SA_RESETHAND : 0;
+
+  int ret = sigaction(signum, &act, &oldact);
+  assert(ret == 0);
+}
+
+void SignalHandler::unregister_handler(int signum, signal_handler_t handler)
+{
+  assert(signum >= 0 && signum < 32);
+  safe_handler *h = handlers[signum];
+  assert(h);
+  assert(h->handler == handler);
+
+  // restore to default
+  signal(signum, SIG_DFL);
+
+  // _then_ remove our handlers entry
+  lock.Lock();
+  handlers[signum] = NULL;
+  lock.Unlock();
+
+  // this will wake up select() so that worker thread sees our handler is gone
+  close(h->pipefd[0]);
+  close(h->pipefd[1]);
+  delete h;
+}
+
+
+// -------
+
+void init_async_signal_handler()
+{
+  assert(!g_signal_handler);
+  g_signal_handler = new SignalHandler;
+}
+
+void shutdown_async_signal_handler()
+{
+  assert(g_signal_handler);
+  delete g_signal_handler;
+  g_signal_handler = NULL;
+}
+
+void register_async_signal_handler(int signum, signal_handler_t handler)
+{
+  assert(g_signal_handler);
+  g_signal_handler->register_handler(signum, handler, false);
+}
+
+void register_async_signal_handler_oneshot(int signum, signal_handler_t handler)
+{
+  assert(g_signal_handler);
+  g_signal_handler->register_handler(signum, handler, true);
+}
+
+void unregister_async_signal_handler(int signum, signal_handler_t handler)
+{
+  assert(g_signal_handler);
+  g_signal_handler->unregister_handler(signum, handler);
+}
+
+
+
diff --git a/src/global/signal_handler.h b/src/global/signal_handler.h
index 1494f32abcb..8acfaed1a4c 100644
--- a/src/global/signal_handler.h
+++ b/src/global/signal_handler.h
@@ -28,4 +28,18 @@ void sighup_handler(int signum);
 // Install the standard Ceph signal handlers
 void install_standard_sighandlers(void);
 
+
+/// initialize async signal handler framework
+void init_async_signal_handler();
+
+/// shutdown async signal handler framework
+void shutdown_async_signal_handler();
+
+/// install a safe, async, callback for the given signal
+void register_async_signal_handler(int signum, signal_handler_t handler);
+void register_async_signal_handler_oneshot(int signum, signal_handler_t handler);
+
+/// uninstall a safe async signal callback
+void unregister_async_signal_handler(int signum, signal_handler_t handler);
+
 #endif
diff --git a/src/test/signals.cc b/src/test/signals.cc
index d68424c1735..43754988559 100644
--- a/src/test/signals.cc
+++ b/src/test/signals.cc
@@ -45,3 +45,68 @@ TEST(SignalApi, SimpleInstallAndTest)
 TEST(SignalEffects, ErrnoTest1)
 {
 }
+
+bool usr1 = false;
+bool usr2 = false;
+
+void reset()
+{
+  usr1 = false;
+  usr2 = false;
+}
+
+void testhandler(int signal)
+{
+  switch (signal) {
+  case SIGUSR1:
+    usr1 = true;
+    break;
+  case SIGUSR2:
+    usr2 = true;
+    break;
+  default:
+    assert(0 == "unexpected signal");
+  }
+}
+
+TEST(SignalHandler, Single)
+{
+  reset();
+  init_async_signal_handler();
+  register_async_signal_handler(SIGUSR1, testhandler);
+  ASSERT_TRUE(usr1 == false);
+
+  int ret = kill(getpid(), SIGUSR1);
+  ASSERT_EQ(ret, 0);
+
+  sleep(1);
+  ASSERT_TRUE(usr1 == true);
+
+  unregister_async_signal_handler(SIGUSR1, testhandler);
+  shutdown_async_signal_handler();
+}
+
+TEST(SignalHandler, Multiple)
+{
+  int ret;
+
+  reset();
+  init_async_signal_handler();
+  register_async_signal_handler(SIGUSR1, testhandler);
+  register_async_signal_handler(SIGUSR2, testhandler);
+  ASSERT_TRUE(usr1 == false);
+  ASSERT_TRUE(usr2 == false);
+
+  ret = kill(getpid(), SIGUSR1);
+  ASSERT_EQ(ret, 0);
+  ret = kill(getpid(), SIGUSR2);
+  ASSERT_EQ(ret, 0);
+
+  sleep(1);
+  ASSERT_TRUE(usr1 == true);
+  ASSERT_TRUE(usr2 == true);
+
+  unregister_async_signal_handler(SIGUSR1, testhandler);
+  unregister_async_signal_handler(SIGUSR2, testhandler);
+  shutdown_async_signal_handler();
+}