test/librados_test_stub: watch/notify should be connection-aware

Utilize the AIO completion thread of the local connection to best
simulate how librados actually performs when connected a cluster.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

2 files changed, 25 insertions, 21 deletions

diff --git a/src/test/librados_test_stub/TestWatchNotify.cc b/src/test/librados_test_stub/TestWatchNotify.cc
index 73383b52654..1f2753aa4cf 100644
--- a/src/test/librados_test_stub/TestWatchNotify.cc
+++ b/src/test/librados_test_stub/TestWatchNotify.cc
@@ -3,6 +3,7 @@
 
 #include "test/librados_test_stub/TestWatchNotify.h"
 #include "include/Context.h"
+#include "include/stringify.h"
 #include "common/Finisher.h"
 #include "test/librados_test_stub/TestRadosClient.h"
 #include <boost/bind.hpp>
@@ -90,6 +91,7 @@ void TestWatchNotify::aio_notify(TestRadosClient *rados_client,
   SharedWatcher watcher = get_watcher(oid);
 
   SharedNotifyHandle notify_handle(new NotifyHandle());
+  notify_handle->rados_client = rados_client;
   notify_handle->pbl = pbl;
   notify_handle->on_notify = on_notify;
   for (auto &watch_handle_pair : watcher->watch_handles) {
@@ -136,6 +138,7 @@ int TestWatchNotify::watch(TestRadosClient *rados_client,
   SharedWatcher watcher = get_watcher(o);
 
   WatchHandle watch_handle;
+  watch_handle.rados_client = rados_client;
   watch_handle.gid = gid;
   watch_handle.handle = ++m_handle;
   watch_handle.watch_ctx = ctx;
@@ -213,22 +216,24 @@ void TestWatchNotify::execute_notify(TestRadosClient *rados_client,
       assert(watch_handle.gid == watcher_id.first);
       assert(watch_handle.handle == watcher_id.second);
 
-      bufferlist notify_bl;
-      notify_bl.append(bl);
-
-      m_lock.Unlock();
-      if (watch_handle.watch_ctx2 != NULL) {
-        watch_handle.watch_ctx2->handle_notify(notify_id, w_it->first, 0,
-                                               notify_bl);
-      } else if (watch_handle.watch_ctx != NULL) {
-        watch_handle.watch_ctx->notify(0, 0, notify_bl);
-      }
-      m_lock.Lock();
-
-      if (watch_handle.watch_ctx2 == NULL) {
-        // auto ack old-style watch/notify clients
-        ack_notify(rados_client, oid, notify_id, watcher_id, bufferlist());
-      }
+      uint64_t notifier_id = rados_client->get_instance_id();
+      watch_handle.rados_client->get_aio_finisher()->queue(new FunctionContext(
+        [this, oid, bl, notify_id, watch_handle, notifier_id](int r) {
+          bufferlist notify_bl;
+          notify_bl.append(bl);
+
+          if (watch_handle.watch_ctx2 != NULL) {
+            watch_handle.watch_ctx2->handle_notify(notify_id,
+                                                   watch_handle.handle,
+                                                   notifier_id, notify_bl);
+          } else if (watch_handle.watch_ctx != NULL) {
+            watch_handle.watch_ctx->notify(0, 0, notify_bl);
+
+            // auto ack old-style watch/notify clients
+            ack_notify(watch_handle.rados_client, oid, notify_id,
+                       {watch_handle.gid, watch_handle.handle}, bufferlist());
+          }
+        }));
     }
   }
 
@@ -299,10 +304,8 @@ void TestWatchNotify::finish_notify(TestRadosClient *rados_client,
     ::encode(notify_handle->pending_watcher_ids, *notify_handle->pbl);
   }
 
-  m_lock.Unlock();
-  notify_handle->on_notify->complete(0);
-  m_lock.Lock();
-
+  notify_handle->rados_client->get_aio_finisher()->queue(
+    notify_handle->on_notify, 0);
   watcher->notify_handles.erase(notify_id);
   if (watcher->watch_handles.empty() && watcher->notify_handles.empty()) {
     m_file_watchers.erase(oid);
diff --git a/src/test/librados_test_stub/TestWatchNotify.h b/src/test/librados_test_stub/TestWatchNotify.h
index bd52add1792..3b0dc50e6cb 100644
--- a/src/test/librados_test_stub/TestWatchNotify.h
+++ b/src/test/librados_test_stub/TestWatchNotify.h
@@ -26,6 +26,7 @@ public:
   typedef std::map<std::pair<uint64_t, uint64_t>, bufferlist> NotifyResponses;
 
   struct NotifyHandle {
+    TestRadosClient *rados_client = nullptr;
     WatcherIDs pending_watcher_ids;
     NotifyResponses notify_responses;
     bufferlist *pbl = nullptr;
@@ -35,7 +36,7 @@ public:
   typedef std::map<uint64_t, SharedNotifyHandle> NotifyHandles;
 
   struct WatchHandle {
-    TestRadosClient *rados_client;
+    TestRadosClient *rados_client = nullptr;
     uint64_t gid;
     uint64_t handle;
     librados::WatchCtx* watch_ctx;