msg/async: Postpone bind if network stack is not ready

RDMAStack shouldn't access hardware from the parent process.
The only reason to do so, is because bind is called before the fork.
After this patch the bind is postponed until the NetworkStack reports
that it is ready to bind.
For NetworkStack types will always return true, except the RDMAStack
which will return true only after the fork (after
AsyncMessenger::ready() is called).

This patch is based on a patch by Haomai Wang <haomai@xsky.com>

Issue: 995322
Change-Id: I1d0d0d52db0a339b9319680c18ee05cde87b2b64
Signed-off-by: Amir Vadai <amir@vadai.me>

4 files changed, 39 insertions, 2 deletions

diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc
index 814d058c8f7..122473179c3 100644
--- a/src/msg/async/AsyncMessenger.cc
+++ b/src/msg/async/AsyncMessenger.cc
@@ -291,6 +291,15 @@ void AsyncMessenger::ready()
 {
   ldout(cct,10) << __func__ << " " << get_myaddr() << dendl;
 
+  stack->ready();
+  if (pending_bind) {
+    int err = bind(pending_bind_addr);
+    if (err) {
+      lderr(cct) << __func__ << " postponed bind failed" << dendl;
+      ceph_abort();
+    }
+  }
+
   Mutex::Locker l(lock);
   for (auto &&p : processors)
     p->start();
@@ -320,12 +329,23 @@ int AsyncMessenger::shutdown()
 int AsyncMessenger::bind(const entity_addr_t &bind_addr)
 {
   lock.Lock();
-  if (started) {
+
+  if (!pending_bind && started) {
     ldout(cct,10) << __func__ << " already started" << dendl;
     lock.Unlock();
     return -1;
   }
+
   ldout(cct,10) << __func__ << " bind " << bind_addr << dendl;
+
+  if (!stack->is_ready()) {
+    ldout(cct, 10) << __func__ << " Network Stack is not ready for bind yet - postponed" << dendl;
+    pending_bind_addr = bind_addr;
+    pending_bind = true;
+    lock.Unlock();
+    return 0;
+  }
+
   lock.Unlock();
 
   // bind to a socket
diff --git a/src/msg/async/AsyncMessenger.h b/src/msg/async/AsyncMessenger.h
index c234a70a442..01af5110495 100644
--- a/src/msg/async/AsyncMessenger.h
+++ b/src/msg/async/AsyncMessenger.h
@@ -237,6 +237,17 @@ private:
   bool need_addr;
 
   /**
+   * set to bind address if bind was called before NetworkStack was ready to
+   * bind
+   */
+  entity_addr_t pending_bind_addr;
+
+  /**
+   * false; set to true if a pending bind exists
+   */
+  bool pending_bind = false;
+
+  /**
    *  The following aren't lock-protected since you shouldn't be able to race
    *  the only writers.
    */
diff --git a/src/msg/async/Stack.h b/src/msg/async/Stack.h
index 2ddfa8c6754..3b7adca35da 100644
--- a/src/msg/async/Stack.h
+++ b/src/msg/async/Stack.h
@@ -287,7 +287,6 @@ class NetworkStack : public CephContext::ForkWatcher {
  protected:
   CephContext *cct;
   vector<Worker*> workers;
-  // Used to indicate whether thread started
 
   explicit NetworkStack(CephContext *c, const string &t);
  public:
@@ -337,6 +336,8 @@ class NetworkStack : public CephContext::ForkWatcher {
     start();
   }
 
+  virtual bool is_ready() { return true; };
+  virtual void ready() { };
 };
 
 #endif //CEPH_MSG_ASYNC_STACK_H
diff --git a/src/msg/async/rdma/RDMAStack.h b/src/msg/async/rdma/RDMAStack.h
index 21355daab38..35eb531a153 100644
--- a/src/msg/async/rdma/RDMAStack.h
+++ b/src/msg/async/rdma/RDMAStack.h
@@ -197,6 +197,8 @@ class RDMAStack : public NetworkStack {
   RDMADispatcher *dispatcher;
   PerfCounters *perf_counter;
 
+  std::atomic<bool> fork_finished = {false};
+
  public:
   explicit RDMAStack(CephContext *cct, const string &t);
   virtual ~RDMAStack();
@@ -206,5 +208,8 @@ class RDMAStack : public NetworkStack {
   virtual void spawn_worker(unsigned i, std::function<void ()> &&func) override;
   virtual void join_worker(unsigned i) override;
   RDMADispatcher *get_dispatcher() { return dispatcher; }
+
+  virtual bool is_ready() override { return fork_finished.load(); };
+  virtual void ready() override { fork_finished = true; };
 };
 #endif