From 57e792bcae285ae02e347740960dbff60b7b21a4 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Mon, 20 Nov 2017 02:04:59 +0800 Subject: bluestore: enable building bluestore w/o libaio KernelDevice is tightly coupled with libaio. more work is needed to decouple aio from it. but by guarding KernelDevice with HAVE_LIBAIO, we can enable bluestore on platforms w/o libaio. Signed-off-by: Kefu Chai --- src/os/CMakeLists.txt | 8 ++++++-- src/os/bluestore/BlockDevice.cc | 8 +++++++- src/os/bluestore/BlockDevice.h | 21 ++++++++++++++++----- src/os/bluestore/BlueFS.cc | 9 ++++++++- src/os/bluestore/BlueFS.h | 2 ++ src/os/bluestore/BlueStore.cc | 7 ++++--- src/os/bluestore/aio.cc | 4 ---- src/os/bluestore/aio.h | 5 ----- 8 files changed, 43 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index 050b02de894..9099695a776 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -29,14 +29,18 @@ if(WITH_BLUESTORE) bluestore/BlueStore.cc bluestore/bluestore_types.cc bluestore/FreelistManager.cc - bluestore/KernelDevice.cc bluestore/StupidAllocator.cc bluestore/BitMapAllocator.cc bluestore/BitAllocator.cc - bluestore/aio.cc ) endif(WITH_BLUESTORE) +if(HAVE_LIBAIO) + list(APPEND libos_srcs + bluestore/KernelDevice.cc + bluestore/aio.cc) +endif() + if(WITH_FUSE) list(APPEND libos_srcs FuseStore.cc) diff --git a/src/os/bluestore/BlockDevice.cc b/src/os/bluestore/BlockDevice.cc index b6f4dd8a103..d82139f031f 100644 --- a/src/os/bluestore/BlockDevice.cc +++ b/src/os/bluestore/BlockDevice.cc @@ -17,7 +17,12 @@ #include #include +#include "BlockDevice.h" + +#if defined(HAVE_LIBAIO) #include "KernelDevice.h" +#endif + #if defined(HAVE_SPDK) #include "NVMEDevice.h" #endif @@ -82,10 +87,11 @@ BlockDevice *BlockDevice::create(CephContext* cct, const string& path, return new PMEMDevice(cct, cb, cbpriv); } #endif - +#if defined(HAVE_LIBAIO) if (type == "kernel") { return new KernelDevice(cct, cb, cbpriv); } +#endif #if defined(HAVE_SPDK) if (type == "ust-nvme") { return new NVMEDevice(cct, cb, cbpriv); diff --git a/src/os/bluestore/BlockDevice.h b/src/os/bluestore/BlockDevice.h index 61b7aee9b43..89b76becb60 100644 --- a/src/os/bluestore/BlockDevice.h +++ b/src/os/bluestore/BlockDevice.h @@ -19,14 +19,24 @@ #include #include -#include #include +#include +#include +#include +#include +#include #include "acconfig.h" +#ifdef HAVE_LIBAIO #include "aio.h" +#endif +#include "include/assert.h" +#include "include/buffer.h" #define SPDK_PREFIX "spdk:" +class CephContext; + /// track in-flight io struct IOContext { private: @@ -43,9 +53,10 @@ public: std::atomic_int total_nseg = {0}; #endif - +#ifdef HAVE_LIBAIO std::list pending_aios; ///< not yet submitted std::list running_aios; ///< submitting or submitted +#endif std::atomic_int num_pending = {0}; std::atomic_int num_running = {0}; bool allow_eio; @@ -128,11 +139,11 @@ public: virtual int collect_metadata(const std::string& prefix, std::map *pm) const = 0; - virtual int get_devname(string *out) { + virtual int get_devname(std::string *out) { return -ENOENT; } - virtual int get_devices(set *ls) { - string s; + virtual int get_devices(std::set *ls) { + std::string s; if (get_devname(&s) == 0) { ls->insert(s); } diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 3f6ba2544ea..45345f5786b 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -1242,12 +1242,14 @@ void BlueFS::_compact_log_sync() log_writer->append(bl); int r = _flush(log_writer, true); assert(r == 0); +#ifdef HAVE_LIBAIO if (!cct->_conf->bluefs_sync_write) { list completed_ios; _claim_completed_aios(log_writer, &completed_ios); wait_for_aio(log_writer); completed_ios.clear(); } +#endif flush_bdev(); dout(10) << __func__ << " writing super" << dendl; @@ -1778,6 +1780,7 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) return 0; } +#ifdef HAVE_LIBAIO // we need to retire old completed aios so they don't stick around in // memory indefinitely (along with their bufferlist refs). void BlueFS::_claim_completed_aios(FileWriter *h, list *ls) @@ -1803,6 +1806,7 @@ void BlueFS::wait_for_aio(FileWriter *h) } dout(10) << __func__ << " " << h << " done in " << (ceph_clock_now() - start) << dendl; } +#endif int BlueFS::_flush(FileWriter *h, bool force) { @@ -1892,6 +1896,7 @@ int BlueFS::_fsync(FileWriter *h, std::unique_lock& l) void BlueFS::_flush_bdev_safely(FileWriter *h) { +#ifdef HAVE_LIBAIO if (!cct->_conf->bluefs_sync_write) { list completed_ios; _claim_completed_aios(h, &completed_ios); @@ -1900,7 +1905,9 @@ void BlueFS::_flush_bdev_safely(FileWriter *h) completed_ios.clear(); flush_bdev(); lock.lock(); - } else { + } else +#endif + { lock.unlock(); flush_bdev(); lock.lock(); diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index d6ca2dc3707..36ce36e59b5 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -272,8 +272,10 @@ private: int _flush(FileWriter *h, bool force); int _fsync(FileWriter *h, std::unique_lock& l); +#ifdef HAVE_LIBAIO void _claim_completed_aios(FileWriter *h, list *ls); void wait_for_aio(FileWriter *h); // safe to call without a lock +#endif int _flush_and_sync_log(std::unique_lock& l, uint64_t want_seq = 0, diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 3c2c348b1c3..fc9c3031740 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -7883,10 +7883,11 @@ void BlueStore::_txc_calc_cost(TransContext *txc) // a configurable (with different hdd and ssd defaults), and add // that to the bytes value. int ios = 1; // one "io" for the kv commit +#ifdef HAVE_LIBAIO for (auto& p : txc->ioc.pending_aios) { ios += p.iov.size(); } - +#endif #ifdef HAVE_SPDK ios += txc->ioc.total_nseg; #endif @@ -8037,10 +8038,10 @@ void BlueStore::_txc_finish_io(TransContext *txc) OpSequencer *osr = txc->osr.get(); std::lock_guard l(osr->qlock); txc->state = TransContext::STATE_IO_DONE; - +#ifdef HAVE_LIBAIO // release aio contexts (including pinned buffers). txc->ioc.running_aios.clear(); - +#endif OpSequencer::q_list_t::iterator p = osr->q.iterator_to(*txc); while (p != osr->q.begin()) { --p; diff --git a/src/os/bluestore/aio.cc b/src/os/bluestore/aio.cc index 79e07115bec..3598702dbfd 100644 --- a/src/os/bluestore/aio.cc +++ b/src/os/bluestore/aio.cc @@ -3,8 +3,6 @@ #include "aio.h" -#if defined(HAVE_LIBAIO) - std::ostream& operator<<(std::ostream& os, const aio_t& aio) { unsigned i = 0; @@ -72,5 +70,3 @@ int aio_queue_t::get_next_completed(int timeout_ms, aio_t **paio, int max) } return r; } - -#endif diff --git a/src/os/bluestore/aio.h b/src/os/bluestore/aio.h index 0be38beb4c5..cae6b320a24 100644 --- a/src/os/bluestore/aio.h +++ b/src/os/bluestore/aio.h @@ -2,9 +2,6 @@ // vim: ts=8 sw=2 smarttab #pragma once - -#include "acconfig.h" -#ifdef HAVE_LIBAIO # include #include @@ -91,5 +88,3 @@ struct aio_queue_t { void *priv, int *retries); int get_next_completed(int timeout_ms, aio_t **paio, int max); }; - -#endif -- cgit v1.2.3