bluestore: enable building bluestore w/o libaio

KernelDevice is tightly coupled with libaio. more work is needed to
decouple aio from it. but by guarding KernelDevice with HAVE_LIBAIO, we
can enable bluestore on platforms w/o libaio.

Signed-off-by: Kefu Chai <kchai@redhat.com>

8 files changed, 43 insertions, 21 deletions

diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt
index 050b02de894..9099695a776 100644
--- a/src/os/CMakeLists.txt
+++ b/src/os/CMakeLists.txt
@@ -29,14 +29,18 @@ if(WITH_BLUESTORE)
     bluestore/BlueStore.cc
     bluestore/bluestore_types.cc
     bluestore/FreelistManager.cc
-    bluestore/KernelDevice.cc
     bluestore/StupidAllocator.cc
     bluestore/BitMapAllocator.cc
     bluestore/BitAllocator.cc
-    bluestore/aio.cc
   )
 endif(WITH_BLUESTORE)
 
+if(HAVE_LIBAIO)
+  list(APPEND libos_srcs
+    bluestore/KernelDevice.cc
+    bluestore/aio.cc)
+endif()
+
 if(WITH_FUSE)
   list(APPEND libos_srcs
     FuseStore.cc)
diff --git a/src/os/bluestore/BlockDevice.cc b/src/os/bluestore/BlockDevice.cc
index b6f4dd8a103..d82139f031f 100644
--- a/src/os/bluestore/BlockDevice.cc
+++ b/src/os/bluestore/BlockDevice.cc
@@ -17,7 +17,12 @@
 #include <libgen.h>
 #include <unistd.h>
 
+#include "BlockDevice.h"
+
+#if defined(HAVE_LIBAIO)
 #include "KernelDevice.h"
+#endif
+
 #if defined(HAVE_SPDK)
 #include "NVMEDevice.h"
 #endif
@@ -82,10 +87,11 @@ BlockDevice *BlockDevice::create(CephContext* cct, const string& path,
     return new PMEMDevice(cct, cb, cbpriv);
   }
 #endif
-
+#if defined(HAVE_LIBAIO)
   if (type == "kernel") {
     return new KernelDevice(cct, cb, cbpriv);
   }
+#endif
 #if defined(HAVE_SPDK)
   if (type == "ust-nvme") {
     return new NVMEDevice(cct, cb, cbpriv);
diff --git a/src/os/bluestore/BlockDevice.h b/src/os/bluestore/BlockDevice.h
index 61b7aee9b43..89b76becb60 100644
--- a/src/os/bluestore/BlockDevice.h
+++ b/src/os/bluestore/BlockDevice.h
@@ -19,14 +19,24 @@
 
 #include <atomic>
 #include <condition_variable>
-#include <mutex>
 #include <list>
+#include <map>
+#include <mutex>
+#include <set>
+#include <string>
+#include <vector>
 
 #include "acconfig.h"
+#ifdef HAVE_LIBAIO
 #include "aio.h"
+#endif
+#include "include/assert.h"
+#include "include/buffer.h"
 
 #define SPDK_PREFIX "spdk:"
 
+class CephContext;
+
 /// track in-flight io
 struct IOContext {
 private:
@@ -43,9 +53,10 @@ public:
   std::atomic_int total_nseg = {0};
 #endif
 
-
+#ifdef HAVE_LIBAIO
   std::list<aio_t> pending_aios;    ///< not yet submitted
   std::list<aio_t> running_aios;    ///< submitting or submitted
+#endif
   std::atomic_int num_pending = {0};
   std::atomic_int num_running = {0};
   bool allow_eio;
@@ -128,11 +139,11 @@ public:
 
   virtual int collect_metadata(const std::string& prefix, std::map<std::string,std::string> *pm) const = 0;
 
-  virtual int get_devname(string *out) {
+  virtual int get_devname(std::string *out) {
     return -ENOENT;
   }
-  virtual int get_devices(set<string> *ls) {
-    string s;
+  virtual int get_devices(std::set<std::string> *ls) {
+    std::string s;
     if (get_devname(&s) == 0) {
       ls->insert(s);
     }
diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
index 3f6ba2544ea..45345f5786b 100644
--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
@@ -1242,12 +1242,14 @@ void BlueFS::_compact_log_sync()
   log_writer->append(bl);
   int r = _flush(log_writer, true);
   assert(r == 0);
+#ifdef HAVE_LIBAIO
   if (!cct->_conf->bluefs_sync_write) {
     list<aio_t> completed_ios;
     _claim_completed_aios(log_writer, &completed_ios);
     wait_for_aio(log_writer);
     completed_ios.clear();
   }
+#endif
   flush_bdev();
 
   dout(10) << __func__ << " writing super" << dendl;
@@ -1778,6 +1780,7 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length)
   return 0;
 }
 
+#ifdef HAVE_LIBAIO
 // we need to retire old completed aios so they don't stick around in
 // memory indefinitely (along with their bufferlist refs).
 void BlueFS::_claim_completed_aios(FileWriter *h, list<aio_t> *ls)
@@ -1803,6 +1806,7 @@ void BlueFS::wait_for_aio(FileWriter *h)
   }
   dout(10) << __func__ << " " << h << " done in " << (ceph_clock_now() - start) << dendl;
 }
+#endif
 
 int BlueFS::_flush(FileWriter *h, bool force)
 {
@@ -1892,6 +1896,7 @@ int BlueFS::_fsync(FileWriter *h, std::unique_lock<std::mutex>& l)
 
 void BlueFS::_flush_bdev_safely(FileWriter *h)
 {
+#ifdef HAVE_LIBAIO
   if (!cct->_conf->bluefs_sync_write) {
     list<aio_t> completed_ios;
     _claim_completed_aios(h, &completed_ios);
@@ -1900,7 +1905,9 @@ void BlueFS::_flush_bdev_safely(FileWriter *h)
     completed_ios.clear();
     flush_bdev();
     lock.lock();
-  } else {
+  } else
+#endif
+  {
     lock.unlock();
     flush_bdev();
     lock.lock();
diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h
index d6ca2dc3707..36ce36e59b5 100644
--- a/src/os/bluestore/BlueFS.h
+++ b/src/os/bluestore/BlueFS.h
@@ -272,8 +272,10 @@ private:
   int _flush(FileWriter *h, bool force);
   int _fsync(FileWriter *h, std::unique_lock<std::mutex>& l);
 
+#ifdef HAVE_LIBAIO
   void _claim_completed_aios(FileWriter *h, list<aio_t> *ls);
   void wait_for_aio(FileWriter *h);  // safe to call without a lock
+#endif
 
   int _flush_and_sync_log(std::unique_lock<std::mutex>& l,
 			  uint64_t want_seq = 0,
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 3c2c348b1c3..fc9c3031740 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -7883,10 +7883,11 @@ void BlueStore::_txc_calc_cost(TransContext *txc)
   // a configurable (with different hdd and ssd defaults), and add
   // that to the bytes value.
   int ios = 1;  // one "io" for the kv commit
+#ifdef HAVE_LIBAIO
   for (auto& p : txc->ioc.pending_aios) {
     ios += p.iov.size();
   }
-
+#endif
 #ifdef HAVE_SPDK
   ios += txc->ioc.total_nseg;
 #endif
@@ -8037,10 +8038,10 @@ void BlueStore::_txc_finish_io(TransContext *txc)
   OpSequencer *osr = txc->osr.get();
   std::lock_guard<std::mutex> l(osr->qlock);
   txc->state = TransContext::STATE_IO_DONE;
-
+#ifdef HAVE_LIBAIO
   // release aio contexts (including pinned buffers).
   txc->ioc.running_aios.clear();
-
+#endif
   OpSequencer::q_list_t::iterator p = osr->q.iterator_to(*txc);
   while (p != osr->q.begin()) {
     --p;
diff --git a/src/os/bluestore/aio.cc b/src/os/bluestore/aio.cc
index 79e07115bec..3598702dbfd 100644
--- a/src/os/bluestore/aio.cc
+++ b/src/os/bluestore/aio.cc
@@ -3,8 +3,6 @@
 
 #include "aio.h"
 
-#if defined(HAVE_LIBAIO)
-
 std::ostream& operator<<(std::ostream& os, const aio_t& aio)
 {
   unsigned i = 0;
@@ -72,5 +70,3 @@ int aio_queue_t::get_next_completed(int timeout_ms, aio_t **paio, int max)
   }
   return r;
 }
-
-#endif
diff --git a/src/os/bluestore/aio.h b/src/os/bluestore/aio.h
index 0be38beb4c5..cae6b320a24 100644
--- a/src/os/bluestore/aio.h
+++ b/src/os/bluestore/aio.h
@@ -2,9 +2,6 @@
 // vim: ts=8 sw=2 smarttab
 
 #pragma once
-
-#include "acconfig.h"
-#ifdef HAVE_LIBAIO
 # include <libaio.h>
 
 #include <boost/intrusive/list.hpp>
@@ -91,5 +88,3 @@ struct aio_queue_t {
 		   void *priv, int *retries);
   int get_next_completed(int timeout_ms, aio_t **paio, int max);
 };
-
-#endif